{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "load_model_ada = 0 \n",
    "load_model_dnn = 0 \n",
    "load_model_mlp = 0 \n",
    "load_model_lgbm = 0 \n",
    "load_model_rf = 0 \n",
    "load_model_svm = 0\n",
    "load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "\n",
    "# load_model_ada = 1\n",
    "# load_model_dnn = 1 \n",
    "# load_model_mlp = 1 \n",
    "# load_model_lgbm = 1 \n",
    "# load_model_rf = 1 \n",
    "# load_model_svm = 1\n",
    "# load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "feature_selection_bit = 0\n",
    "# feature_selection_bit = 1\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "    output_file_name = \"ensemble_base_models_all_features.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---- ensemble_base_models_all_features', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    output_file_name = \"ensemble_base_models_feature_selection.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('----ensemble_base_models_feature_selection--', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/oarreche@ads.iu.edu/anaconda3/envs/HITL/lib/python3.6/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import time\n",
    "start_program = time.time()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "'''\n",
    "all features\n",
    "'''\n",
    "\n",
    "req_cols = ['FLOW_DURATION_MILLISECONDS','FIRST_SWITCHED',\n",
    "            'TOTAL_FLOWS_EXP','TCP_WIN_MSS_IN','LAST_SWITCHED',\n",
    "            'TCP_WIN_MAX_IN','TCP_WIN_MIN_IN','TCP_WIN_MIN_OUT',\n",
    "           'PROTOCOL','TCP_WIN_MAX_OUT','TCP_FLAGS',\n",
    "            'TCP_WIN_SCALE_OUT','TCP_WIN_SCALE_IN','SRC_TOS',\n",
    "            'DST_TOS','FLOW_ID','L4_SRC_PORT','L4_DST_PORT',\n",
    "           'MIN_IP_PKT_LEN','MAX_IP_PKT_LEN','TOTAL_PKTS_EXP',\n",
    "           'TOTAL_BYTES_EXP','IN_BYTES','IN_PKTS','OUT_BYTES','OUT_PKTS',\n",
    "            'ALERT']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading Database\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "address = '/home/oarreche@ads.iu.edu/HITL/sensor/sensor_db'\n",
    "print('Loading Database')\n",
    "print('--------------------------------------------------')\n",
    "\n",
    "fraction = 1\n",
    "#Denial of Service\n",
    "df0 = pd.read_csv (address + '/dos-03-15-2022-15-44-32.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df1 = pd.read_csv (address + '/dos-03-16-2022-13-45-18.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df2 = pd.read_csv (address + '/dos-03-17-2022-16-22-53.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df3 = pd.read_csv (address + '/dos-03-18-2022-19-27-05.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df4 = pd.read_csv (address + '/dos-03-19-2022-20-01-53.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df5 = pd.read_csv (address + '/dos-03-20-2022-14-27-54.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "#Malware\n",
    "#df6 = pd.read_csv ('sensor_db/malware-03-25-2022-17-57-07.csv', usecols=req_cols)\n",
    "\n",
    "#Normal\n",
    "df7 = pd.read_csv  (address + '/normal-03-15-2022-15-43-44.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df8 = pd.read_csv  (address + '/normal-03-16-2022-13-44-27.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df9 = pd.read_csv  (address + '/normal-03-17-2022-16-21-30.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df10 = pd.read_csv (address + '/normal-03-18-2022-19-17-31.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df11 = pd.read_csv (address + '/normal-03-18-2022-19-25-48.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df12 = pd.read_csv (address + '/normal-03-19-2022-20-01-16.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df13 = pd.read_csv (address + '/normal-03-20-2022-14-27-30.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "#PortScanning\n",
    "df14 = pd.read_csv  (address + '/portscanning-03-15-2022-15-44-06.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df15 = pd.read_csv  (address + '/portscanning-03-16-2022-13-44-50.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df16 = pd.read_csv  (address + '/portscanning-03-17-2022-16-22-53.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df17 = pd.read_csv  (address + '/portscanning-03-18-2022-19-27-05.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df18 = pd.read_csv  (address + '/portscanning-03-19-2022-20-01-45.csv', usecols=req_cols).sample(frac = fraction)\n",
    "df19 = pd.read_csv  (address + '/portscanning-03-20-2022-14-27-49.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "frames = [df0, df1, df2, df3, df4, df5, df7, df8, df9, df10, df11, df12, df13, df14, df15, df16, df17, df18, df19]\n",
    "df = pd.concat(frames,ignore_index=True)\n",
    "\n",
    "# shuffle the DataFrame rows\n",
    "df = df.sample(frac =1)\n",
    "\n",
    "# assign alert column to y\n",
    "y = df.pop('ALERT')\n",
    "\n",
    "# join alert back to df\n",
    "df = df.assign( ALERT = y) \n",
    "\n",
    "#Fill NaN with 0s\n",
    "df = df.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    # voting_acc_01 = Acc\n",
    "    # voting_pre_01 = Precision\n",
    "    # weighed_avg_rec_01 = Recall\n",
    "    # weighed_avg_f1_01 = F1\n",
    "    # weighed_avg_bacc_01 = BACC\n",
    "    # weighed_avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from imblearn.over_sampling import RandomOverSampler\n",
    "# from sklearn.datasets import make_classification\n",
    "\n",
    "# # Assuming you have features X and labels Y\n",
    "# # X, Y = make_classification()\n",
    "\n",
    "# ros = RandomOverSampler(sampling_strategy='minority', random_state=100)\n",
    "\n",
    "# X_train, Y_train = ros.fit_resample(X_train, Y_train)\n",
    "\n",
    "\n",
    "# # In[33]:\n",
    "\n",
    "\n",
    "# print(Y_test)\n",
    "\n",
    "\n",
    "# # In[34]:\n",
    "\n",
    "\n",
    "# X_train.values\n",
    "\n",
    "\n",
    "# # In[35]:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#fix \n",
    "\n",
    "if feature_selection_bit == 1:\n",
    "    feature_selection = [\n",
    "                        'dst_host_same_srv_rate',\n",
    "                        'dst_host_srv_count',\n",
    "                        'dst_host_same_src_port_rate',\n",
    "                        'logged_in',\n",
    "                        'dst_host_serror_rate',\n",
    "                        'count',\n",
    "                        'srv_count',\n",
    "                        'dst_host_rerror_rate',\n",
    "                        'Label'\n",
    "                        ]\n",
    "\n",
    "    df_og = df\n",
    "    df = df[feature_selection]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# y = df.pop('Label')\n",
    "# X = df\n",
    "\n",
    "# y1, y2 = pd.factorize(y)\n",
    "\n",
    "# y_0 = pd.DataFrame(y1)\n",
    "# y_1 = pd.DataFrame(y1)\n",
    "# y_2 = pd.DataFrame(y1)\n",
    "# y_3 = pd.DataFrame(y1)\n",
    "# y_4 = pd.DataFrame(y1)\n",
    "\n",
    "\n",
    "# # y_0 = y_0.replace(0, 0)\n",
    "# # y_0 = y_0.replace(1, 1)\n",
    "# y_0 = y_0.replace(2, 1)\n",
    "# y_0 = y_0.replace(3, 1)\n",
    "# y_0 = y_0.replace(4, 1)\n",
    "\n",
    "\n",
    "# y_1 = y_1.replace(1, 999)\n",
    "# y_1 = y_1.replace(0, 1)\n",
    "# # y_1 = y_1.replace(1, 0)\n",
    "# y_1 = y_1.replace(2, 1)\n",
    "# y_1 = y_1.replace(3, 1)\n",
    "# y_1 = y_1.replace(4, 1)\n",
    "# y_1 = y_1.replace(999, 1)\n",
    "\n",
    "\n",
    "# y_2 = y_2.replace(0, 1)\n",
    "# y_2 = y_2.replace(1, 1)\n",
    "# y_2 = y_2.replace(2, 0)\n",
    "# y_2 = y_2.replace(3, 1)\n",
    "# y_2 = y_2.replace(4, 1)\n",
    "\n",
    "\n",
    "# y_3 = y_3.replace(0, 1)\n",
    "# # y_3 = y_3.replace(1, 1)\n",
    "# y_3 = y_3.replace(2, 1)\n",
    "# y_3 = y_3.replace(3, 0)\n",
    "# y_3 = y_3.replace(4, 1)\n",
    "\n",
    "\n",
    "# y_4 = y_4.replace(0, 1)\n",
    "# # y_4 = y_4.replace(1, 1)\n",
    "# y_4 = y_4.replace(2, 1)\n",
    "# y_4 = y_4.replace(3, 1)\n",
    "# y_4 = y_4.replace(4, 0)\n",
    "\n",
    "\n",
    "\n",
    "# df = df.assign(Label = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.pop('ALERT')\n",
    "X = df\n",
    "df = df.assign(Label = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = df.pop('Label')\n",
    "y_numbers, y_label = pd.factorize(y)\n",
    "X = df\n",
    "df = df.assign(Label = y_numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ... 1 0 0]\n",
      "Index(['None', 'Port Scanning', 'Denial of Service'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(y_numbers)\n",
    "print(y_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Divide the dataset between level 00 and level 01\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "split = 0.7 # 0.5\n",
    "\n",
    "# X_00,X_01, y_00, y_01 = sklearn.model_selection.train_test_split(X, y, train_size=split)\n",
    "X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y_numbers, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'None': 19552134, 'Denial of Service': 6425161, 'Port Scanning': 4170392})\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "label_counts2 = Counter(y)\n",
    "print(label_counts2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Base learner Split\n",
    "# split = 0.7\n",
    "\n",
    "# X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X_00, y_00, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FLOW_ID</th>\n",
       "      <th>L4_SRC_PORT</th>\n",
       "      <th>L4_DST_PORT</th>\n",
       "      <th>FIRST_SWITCHED</th>\n",
       "      <th>FLOW_DURATION_MILLISECONDS</th>\n",
       "      <th>LAST_SWITCHED</th>\n",
       "      <th>PROTOCOL</th>\n",
       "      <th>TCP_FLAGS</th>\n",
       "      <th>TCP_WIN_MAX_IN</th>\n",
       "      <th>TCP_WIN_MAX_OUT</th>\n",
       "      <th>...</th>\n",
       "      <th>DST_TOS</th>\n",
       "      <th>TOTAL_FLOWS_EXP</th>\n",
       "      <th>MIN_IP_PKT_LEN</th>\n",
       "      <th>MAX_IP_PKT_LEN</th>\n",
       "      <th>TOTAL_PKTS_EXP</th>\n",
       "      <th>TOTAL_BYTES_EXP</th>\n",
       "      <th>IN_BYTES</th>\n",
       "      <th>IN_PKTS</th>\n",
       "      <th>OUT_BYTES</th>\n",
       "      <th>OUT_PKTS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>14248895</th>\n",
       "      <td>359334585</td>\n",
       "      <td>55703</td>\n",
       "      <td>443</td>\n",
       "      <td>1647544962</td>\n",
       "      <td>74</td>\n",
       "      <td>1647544962</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>1028</td>\n",
       "      <td>8192</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>359334585</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>183</td>\n",
       "      <td>2</td>\n",
       "      <td>213</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7931071</th>\n",
       "      <td>339042927</td>\n",
       "      <td>3478</td>\n",
       "      <td>3478</td>\n",
       "      <td>1647419012</td>\n",
       "      <td>0</td>\n",
       "      <td>1647419012</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>339042927</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15298998</th>\n",
       "      <td>364685755</td>\n",
       "      <td>59906</td>\n",
       "      <td>8282</td>\n",
       "      <td>1647612892</td>\n",
       "      <td>47</td>\n",
       "      <td>1647612892</td>\n",
       "      <td>6</td>\n",
       "      <td>16</td>\n",
       "      <td>510</td>\n",
       "      <td>259</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>364685755</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>41</td>\n",
       "      <td>1</td>\n",
       "      <td>52</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17624651</th>\n",
       "      <td>352050932</td>\n",
       "      <td>3544</td>\n",
       "      <td>53009</td>\n",
       "      <td>1647446720</td>\n",
       "      <td>18493</td>\n",
       "      <td>1647446739</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>352050932</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>672</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2392433</th>\n",
       "      <td>363701817</td>\n",
       "      <td>36402</td>\n",
       "      <td>443</td>\n",
       "      <td>1647603970</td>\n",
       "      <td>3039</td>\n",
       "      <td>1647603973</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>64240</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>363701817</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>180</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14207385</th>\n",
       "      <td>351486510</td>\n",
       "      <td>39086</td>\n",
       "      <td>6443</td>\n",
       "      <td>1647440862</td>\n",
       "      <td>0</td>\n",
       "      <td>1647440862</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>64240</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>351486510</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12646766</th>\n",
       "      <td>350799860</td>\n",
       "      <td>21731</td>\n",
       "      <td>53</td>\n",
       "      <td>1647434885</td>\n",
       "      <td>16</td>\n",
       "      <td>1647434885</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>350799860</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>144</td>\n",
       "      <td>2</td>\n",
       "      <td>272</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7097935</th>\n",
       "      <td>335273273</td>\n",
       "      <td>57798</td>\n",
       "      <td>443</td>\n",
       "      <td>1647349872</td>\n",
       "      <td>9879</td>\n",
       "      <td>1647349882</td>\n",
       "      <td>6</td>\n",
       "      <td>20</td>\n",
       "      <td>1026</td>\n",
       "      <td>166</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>335273273</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>81</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10827417</th>\n",
       "      <td>351453489</td>\n",
       "      <td>64873</td>\n",
       "      <td>443</td>\n",
       "      <td>1647440562</td>\n",
       "      <td>1075</td>\n",
       "      <td>1647440563</td>\n",
       "      <td>6</td>\n",
       "      <td>27</td>\n",
       "      <td>64240</td>\n",
       "      <td>65535</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>351453489</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>62293</td>\n",
       "      <td>69</td>\n",
       "      <td>12355</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2098832</th>\n",
       "      <td>356681274</td>\n",
       "      <td>49834</td>\n",
       "      <td>443</td>\n",
       "      <td>1647516043</td>\n",
       "      <td>3035</td>\n",
       "      <td>1647516046</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>64240</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>356681274</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>180</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21103380 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            FLOW_ID  L4_SRC_PORT  L4_DST_PORT  FIRST_SWITCHED  \\\n",
       "14248895  359334585        55703          443      1647544962   \n",
       "7931071   339042927         3478         3478      1647419012   \n",
       "15298998  364685755        59906         8282      1647612892   \n",
       "17624651  352050932         3544        53009      1647446720   \n",
       "2392433   363701817        36402          443      1647603970   \n",
       "...             ...          ...          ...             ...   \n",
       "14207385  351486510        39086         6443      1647440862   \n",
       "12646766  350799860        21731           53      1647434885   \n",
       "7097935   335273273        57798          443      1647349872   \n",
       "10827417  351453489        64873          443      1647440562   \n",
       "2098832   356681274        49834          443      1647516043   \n",
       "\n",
       "          FLOW_DURATION_MILLISECONDS  LAST_SWITCHED  PROTOCOL  TCP_FLAGS  \\\n",
       "14248895                          74     1647544962         6         24   \n",
       "7931071                            0     1647419012        17          0   \n",
       "15298998                          47     1647612892         6         16   \n",
       "17624651                       18493     1647446739        17          0   \n",
       "2392433                         3039     1647603973         6          2   \n",
       "...                              ...            ...       ...        ...   \n",
       "14207385                           0     1647440862         6          2   \n",
       "12646766                          16     1647434885        17          0   \n",
       "7097935                         9879     1647349882         6         20   \n",
       "10827417                        1075     1647440563         6         27   \n",
       "2098832                         3035     1647516046         6          2   \n",
       "\n",
       "          TCP_WIN_MAX_IN  TCP_WIN_MAX_OUT  ...  DST_TOS  TOTAL_FLOWS_EXP  \\\n",
       "14248895            1028             8192  ...        0        359334585   \n",
       "7931071                0                0  ...        0        339042927   \n",
       "15298998             510              259  ...        0        364685755   \n",
       "17624651               0                0  ...        0        352050932   \n",
       "2392433            64240                0  ...        0        363701817   \n",
       "...                  ...              ...  ...      ...              ...   \n",
       "14207385           64240                0  ...        0        351486510   \n",
       "12646766               0                0  ...        0        350799860   \n",
       "7097935             1026              166  ...        0        335273273   \n",
       "10827417           64240            65535  ...        0        351453489   \n",
       "2098832            64240                0  ...        0        356681274   \n",
       "\n",
       "          MIN_IP_PKT_LEN  MAX_IP_PKT_LEN  TOTAL_PKTS_EXP  TOTAL_BYTES_EXP  \\\n",
       "14248895               0               0               0                0   \n",
       "7931071                0               0               0                0   \n",
       "15298998               0               0               0                0   \n",
       "17624651               0               0               0                0   \n",
       "2392433                0               0               0                0   \n",
       "...                  ...             ...             ...              ...   \n",
       "14207385               0               0               0                0   \n",
       "12646766               0               0               0                0   \n",
       "7097935                0               0               0                0   \n",
       "10827417               0               0               0                0   \n",
       "2098832                0               0               0                0   \n",
       "\n",
       "          IN_BYTES  IN_PKTS  OUT_BYTES  OUT_PKTS  \n",
       "14248895       183        2        213         1  \n",
       "7931071         36        1          0         0  \n",
       "15298998        41        1         52         1  \n",
       "17624651       672        8          0         0  \n",
       "2392433        180        3          0         0  \n",
       "...            ...      ...        ...       ...  \n",
       "14207385        60        1          0         0  \n",
       "12646766       144        2        272         2  \n",
       "7097935         81        2         40         1  \n",
       "10827417     62293       69      12355        44  \n",
       "2098832        180        3          0         0  \n",
       "\n",
       "[21103380 rows x 26 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 2])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LEVEL 0 - Weak models - Base Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 3)                 81        \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 5)                 20        \n",
      "=================================================================\n",
      "Total params: 149\n",
      "Trainable params: 149\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------START of WEAK LEARNERS (BASE MODELS) - STACK 00 -----------------', file = f)\n",
    "\n",
    "#Defining Basemodels\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "\n",
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "\n",
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "\n",
    "#DNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "# #Model Parameters\n",
    "# dropout_rate = 0.01\n",
    "# nodes = 70\n",
    "# out_layer = 5\n",
    "# optimizer='adam'\n",
    "# loss='sparse_categorical_crossentropy'\n",
    "# epochs=1\n",
    "# batch_size=2*256\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train.shape[1]\n",
    "\n",
    "dnn = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "dnn.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn.summary()\n",
    "\n",
    "\n",
    "\n",
    "# dnn = Sequential()\n",
    "# dnn.add(Dense(128, input_dim=X_train.shape[1], activation='relu'))  # Input layer\n",
    "# dnn.add(Dense(64, activation='relu'))  # Hidden layer\n",
    "# dnn.add(Dense(5))  # Output layer\n",
    "\n",
    "# dnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# # summary of model layers\n",
    "# dnn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #SVM\n",
    "# # Wrap SGDClassifier with MultiOutputClassifier\n",
    "# multi_target_clf = MultiOutputClassifier(clf)\n",
    "\n",
    "# # Fit the model on the training data\n",
    "# multi_target_clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "# y_pred = clf.predict(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training SVM\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#Training Basemodels\n",
    "import joblib\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "n_splits = 5  # You can adjust the number of folds as needed\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Training Model')\n",
    "with open(output_file_name, \"a\") as f: print('Training weak models - level 0', file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1 and load_model_ada == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    ada_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    joblib.dump(ada, 'ada_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_rf == 1 and load_model_rf == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf = rf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(model_rf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    rf_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(model_rf, 'rf_base_model.joblib')\n",
    "\n",
    "if use_model_svm == 1 and load_model_svm == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "\n",
    "    start = time.time()\n",
    "    clf.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "    # clf.score(X_train, y_train)\n",
    "    svm_tr_time_taken= time_taken = end - start\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(clf, 'svm_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_knn == 1 and load_model_knn == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    knn_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(knn_clf, 'knn_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_lgbm == 1 and load_model_lgbm == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    lgbm_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(lgbm, 'lgbm_base_model.joblib')\n",
    "\n",
    "if use_model_mlp == 1 and load_model_mlp == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    mlp_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    joblib.dump(MLP, 'mlp_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_dnn == 1 and load_model_dnn == 0:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)\n",
    "    dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(dnn, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    dnn_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    dnn.save(\"DNN_base_model.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from keras.models import Sequential\n",
    "# from keras.layers import Dense\n",
    "# from keras.wrappers.scikit_learn import KerasClassifier\n",
    "# from sklearn.model_selection import GridSearchCV\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# # Define your Keras model as a function\n",
    "# def create_model(optimizer='adam', hidden_layer_size=16):\n",
    "#     # model = Sequential()\n",
    "#     # model.add(Dense(hidden_layer_size, input_dim=input_size, activation='relu'))\n",
    "#     # model.add(Dense(1, activation='sigmoid'))\n",
    "#     # model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "        \n",
    "#     dnn = tf.keras.Sequential()\n",
    "\n",
    "#     # Input layer\n",
    "#     dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "#     # Dense layers with dropout\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     # Output layer\n",
    "#     dnn.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "\n",
    "\n",
    "#     dnn.compile(optimizer=optimizer, loss=loss)\n",
    "\n",
    "#     dnn.summary()\n",
    "#     return dnn\n",
    "\n",
    "# # Create a KerasClassifier\n",
    "# dnn = KerasClassifier(build_fn=create_model, epochs=10, batch_size=32, verbose=0)\n",
    "\n",
    "# # Define the parameter grid for GridSearchCV\n",
    "# param_grid = {\n",
    "#     'optimizer': ['adam', 'sgd'],\n",
    "#     'hidden_layer_size': [8, 16, 32]\n",
    "# }\n",
    "\n",
    "# # Create the StratifiedKFold\n",
    "# cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# # Create GridSearchCV\n",
    "# grid = GridSearchCV(estimator=dnn, param_grid=param_grid, cv=cv, scoring='accuracy')\n",
    "# grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "# # Print the best parameters and best accuracy\n",
    "# print(\"Best Parameters: \", grid_result.best_params_)\n",
    "# print(\"Best Accuracy: \", grid_result.best_score_)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stratified_kfold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading Models\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "if load_model_ada == 1:\n",
    "    ada = joblib.load('ada_base_model.joblib')\n",
    "\n",
    "if load_model_svm == 1:\n",
    "    clf =  joblib.load('svm_base_model.joblib')\n",
    "\n",
    "if load_model_dnn == 1:\n",
    "    dnn = load_model(\"DNN_base_model.h5\")\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf = joblib.load('knn_base_model.joblib')\n",
    "\n",
    "if load_model_mlp == 1:\n",
    "    MLP = joblib.load('mlp_base_model.joblib')\n",
    "\n",
    "if load_model_rf == 1:\n",
    "    rf = joblib.load('rf_base_model.joblib')\n",
    "\n",
    "if load_model_lgbm == 1:\n",
    "    lgbm = joblib.load('lgbm_base_model.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions on the test data\n",
    "# preds_svm = clf.predict(X_test)\n",
    "\n",
    "\n",
    "\n",
    "# y_scores = y_pred\n",
    "# y_true = y_test\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base leaners predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction SVM\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction DNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction KNN\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "with open(output_file_name, \"a\") as f: print('Generating Predictions', file = f)\n",
    "\n",
    "if use_model_rf == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf = rf.predict(X_test)\n",
    "    preds_rf_prob = rf.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    rf_pr_time_taken=  time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_svm == 1:\n",
    "\n",
    "    print('Prediction SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "    start = time.time()\n",
    "    preds_svm = clf.predict(X_test)\n",
    "    # preds_svm_prob = clf.predict_proba(X_test)\n",
    "\n",
    "    #Since SVM does not deal with prob by nature we use a meta learner\n",
    "    # https://stackoverflow.com/questions/55250963/how-to-get-probabilities-for-sgdclassifier-linearsvm\n",
    "\n",
    "    model = CalibratedClassifierCV(clf)\n",
    "\n",
    "    model.fit(X, y)\n",
    "    preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "    end = time.time()\n",
    "    svm_pr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm = lgbm.predict(X_test)\n",
    "    preds_lgbm_prob = lgbm.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    lgbm_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_dnn == 1:\n",
    "\n",
    "    print('Prediction DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #DNN\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn.predict(X_test)\n",
    "    preds_dnn_prob = pred_dnn\n",
    "    preds_dnn = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    dnn_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada = ada.predict(X_test)\n",
    "    preds_ada_prob = ada.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    ada_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_mlp == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_prob = y_pred\n",
    "    preds_mlp = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    mlp_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf.predict(X_test)\n",
    "    preds_knn_prob =knn_clf.predict_proba(X_test)\n",
    "\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    knn_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.calibration import CalibratedClassifierCV\n",
    "# model = CalibratedClassifierCV(clf)\n",
    "\n",
    "# model.fit(X, y)\n",
    "# preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "# print(preds_ada_prob)\n",
    "# print(preds_knn_prob)\n",
    "# print(preds_dnn_prob)\n",
    "# print(preds_mlp_prob)\n",
    "# print(preds_rf_prob)\n",
    "# print(preds_svm_prob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7.86755451e-01 9.15808640e-02 7.78774782e-02 4.31012582e-02\n",
      "  6.84948135e-04]\n",
      " [2.62305098e-01 2.71329463e-02 6.45857927e-01 6.23936481e-02\n",
      "  2.31038051e-03]\n",
      " [1.69951428e-02 9.10446456e-01 5.10282199e-02 2.14799179e-02\n",
      "  5.02632551e-05]\n",
      " ...\n",
      " [9.42972082e-01 3.31526500e-02 1.34223706e-03 2.24056309e-02\n",
      "  1.27400343e-04]\n",
      " [9.12741440e-01 3.97164126e-02 1.80198033e-02 2.95111286e-02\n",
      "  1.12159303e-05]\n",
      " [5.57736620e-01 3.72736907e-01 5.55977328e-02 1.35376604e-02\n",
      "  3.91079556e-04]]\n",
      "[0 2 1 ... 0 0 0]\n",
      "[0 0 0 ... 0 1 0]\n"
     ]
    }
   ],
   "source": [
    "print(preds_svm_prob)\n",
    "preds_3 = np.argmax(preds_svm_prob,axis = 1)\n",
    "print(preds_3)\n",
    "\n",
    "print(preds_svm)\n",
    "# print(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### METRICS - Base Learners"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "\n",
    "\n",
    "# >>> \n",
    "# >>> roc_auc_score(y, clf.predict_proba(X)[:, 1])\n",
    "# 0.99...\n",
    "# >>> roc_auc_score(y, clf.decision_function(X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_test\n",
    "# pred_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2     3    4\n",
      "0  22618.0    186.0   326.0   5.0  0.0\n",
      "1   1091.0  14880.0    57.0   4.0  0.0\n",
      "2    583.0    157.0  3455.0   6.0  0.0\n",
      "3   1071.0      6.0    13.0  70.0  0.0\n",
      "4     25.0      1.0     0.0   2.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9207065266181883\n",
      "Precision total:  0.7139359288591236\n",
      "Recall total:  0.5577129359151647\n",
      "F1 total:  0.570930941536178\n",
      "BACC total:  0.5577129359151647\n",
      "MCC total:  0.8648011689005226\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#RF\n",
    "if use_model_rf == 1:\n",
    "    # start = time.time()\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('RF base model', file = f)\n",
    "\n",
    "    pred_label = preds_rf\n",
    "    name = 'rf'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    globals()[f\"{name}_time_00\"] = rf_pr_time_taken + rf_tr_time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1    2    3    4\n",
      "0  22402.0    733.0  0.0  0.0  0.0\n",
      "1   1195.0  14837.0  0.0  0.0  0.0\n",
      "2   2011.0   2190.0  0.0  0.0  0.0\n",
      "3    998.0    162.0  0.0  0.0  0.0\n",
      "4     20.0      8.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.8357796929706437\n",
      "Precision total:  0.33377077762494667\n",
      "Recall total:  0.37875559611272375\n",
      "F1 total:  0.3548250149879311\n",
      "BACC total:  0.37875559611272375\n",
      "MCC total:  0.7138421055611104\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "if use_model_dnn == 1:\n",
    "    start = time.time()\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('DNN base model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_dnn\n",
    "    name = 'dnn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = dnn_pr_time_taken + dnn_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3    4\n",
      "0  18457.0   3295.0   837.0  546.0  0.0\n",
      "1    412.0  13896.0  1654.0   70.0  0.0\n",
      "2    446.0    862.0  2834.0   59.0  0.0\n",
      "3    649.0     42.0    10.0  459.0  0.0\n",
      "4     13.0      1.0     1.0   13.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8000269323996768\n",
      "Precision total:  0.524620164370353\n",
      "Recall total:  0.5469705911035418\n",
      "F1 total:  0.5325634322997368\n",
      "BACC total:  0.5469705911035418\n",
      "MCC total:  0.6762062856302573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#ADA\n",
    "if use_model_ada == 1:\n",
    "    start = time.time()\n",
    "    \n",
    "    pred_label = preds_ada\n",
    "    name = 'ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = ada_pr_time_taken + ada_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2    3    4\n",
      "0  21584.0    501.0  1046.0  4.0  0.0\n",
      "1   1278.0  14592.0   162.0  0.0  0.0\n",
      "2    563.0   1232.0  2406.0  0.0  0.0\n",
      "3    969.0     65.0   125.0  1.0  0.0\n",
      "4     19.0      6.0     3.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.8659439806086723\n",
      "Precision total:  0.5234127907905088\n",
      "Recall total:  0.48334424220076777\n",
      "F1 total:  0.4830757472142146\n",
      "BACC total:  0.48334424220076777\n",
      "MCC total:  0.7683805085498067\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "if use_model_svm == 1:\n",
    "    start = time.time()\n",
    "\n",
    "    pred_label = preds_svm\n",
    "    name = 'svm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = svm_pr_time_taken + svm_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3    4\n",
      "0  22601.0    175.0   148.0  209.0  2.0\n",
      "1     97.0  15909.0    14.0   12.0  0.0\n",
      "2    156.0     17.0  4006.0   22.0  0.0\n",
      "3    158.0      2.0    12.0  988.0  0.0\n",
      "4     18.0      2.0     0.0    8.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9763892629499955\n",
      "Precision total:  0.7449984935107337\n",
      "Recall total:  0.7549105104158105\n",
      "F1 total:  0.7497724321771452\n",
      "BACC total:  0.7549105104158105\n",
      "MCC total:  0.9601267086185002\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "if use_model_knn == 1:\n",
    "    start = time.time()\n",
    "    pred_label = preds_knn\n",
    "    name = 'knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = knn_pr_time_taken + knn_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3    4\n",
      "0  22448.0    227.0   166.0  292.0  2.0\n",
      "1    153.0  15800.0    44.0   35.0  0.0\n",
      "2    261.0     19.0  3887.0   34.0  0.0\n",
      "3    168.0      5.0    12.0  975.0  0.0\n",
      "4     17.0      2.0     0.0    9.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9675464583894425\n",
      "Precision total:  0.7258257315368958\n",
      "Recall total:  0.7443213616075777\n",
      "F1 total:  0.7341960134322133\n",
      "BACC total:  0.7443213616075777\n",
      "MCC total:  0.9452297110268977\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "if use_model_mlp == 1:\n",
    "    start = time.time()\n",
    "    pred_label = preds_mlp\n",
    "    name = 'mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = mlp_pr_time_taken + mlp_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3     4\n",
      "0  22647.0    154.0   142.0  163.0  29.0\n",
      "1    136.0  15859.0    21.0   10.0   6.0\n",
      "2    111.0     16.0  4056.0   15.0   3.0\n",
      "3    140.0      1.0    11.0  993.0  15.0\n",
      "4     17.0      0.0     0.0    9.0   2.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9775787772690546\n",
      "Precision total:  0.7602977626115522\n",
      "Recall total:  0.7722125926687193\n",
      "F1 total:  0.7650840349094176\n",
      "BACC total:  0.7722125926687193\n",
      "MCC total:  0.9621489670045353\n"
     ]
    }
   ],
   "source": [
    "#lgbm\n",
    "start_lgbm = time.time()\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    pred_label = preds_lgbm\n",
    "    name = 'lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_00\"] = lgbm_pr_time_taken + lgbm_tr_time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3     4\n",
      "0  22672.0    148.0   131.0  175.0   9.0\n",
      "1    182.0  15811.0    33.0    4.0   2.0\n",
      "2    163.0     18.0  3998.0   22.0   0.0\n",
      "3    169.0      1.0    11.0  968.0  11.0\n",
      "4     13.0      0.0     0.0    8.0   7.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9753119669629231\n",
      "Precision total:  0.7977409184547158\n",
      "Recall total:  0.8004726065958696\n",
      "F1 total:  0.7990798000801449\n",
      "BACC total:  0.8004726065958696\n",
      "MCC total:  0.9582370844257531\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train, y_train)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test)\n",
    "\n",
    "\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.3063996\ttest: 1.3060933\tbest: 1.3060933 (0)\ttotal: 28.5ms\tremaining: 2.82s\n",
      "10:\tlearn: 0.4649801\ttest: 0.4648494\tbest: 0.4648494 (10)\ttotal: 249ms\tremaining: 2.01s\n",
      "20:\tlearn: 0.2697143\ttest: 0.2699449\tbest: 0.2699449 (20)\ttotal: 471ms\tremaining: 1.77s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30:\tlearn: 0.1896072\ttest: 0.1904097\tbest: 0.1904097 (30)\ttotal: 690ms\tremaining: 1.53s\n",
      "40:\tlearn: 0.1534307\ttest: 0.1542707\tbest: 0.1542707 (40)\ttotal: 869ms\tremaining: 1.25s\n",
      "50:\tlearn: 0.1330325\ttest: 0.1340119\tbest: 0.1340119 (50)\ttotal: 1.05s\tremaining: 1s\n",
      "60:\tlearn: 0.1187662\ttest: 0.1200194\tbest: 0.1200194 (60)\ttotal: 1.26s\tremaining: 805ms\n",
      "70:\tlearn: 0.1105833\ttest: 0.1120242\tbest: 0.1120242 (70)\ttotal: 1.48s\tremaining: 605ms\n",
      "80:\tlearn: 0.1033562\ttest: 0.1050578\tbest: 0.1050578 (80)\ttotal: 1.7s\tremaining: 398ms\n",
      "90:\tlearn: 0.0981549\ttest: 0.1000457\tbest: 0.1000457 (90)\ttotal: 1.94s\tremaining: 191ms\n",
      "99:\tlearn: 0.0935912\ttest: 0.0954666\tbest: 0.0954666 (99)\ttotal: 2.16s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.09546660735\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3    4\n",
      "0  22568.0    173.0   193.0  201.0  0.0\n",
      "1    190.0  15759.0    71.0   12.0  0.0\n",
      "2    196.0     26.0  3961.0   18.0  0.0\n",
      "3    219.0      7.0    21.0  913.0  0.0\n",
      "4     15.0      0.0     0.0   13.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9695888320316007\n",
      "Precision total:  0.7364693151486311\n",
      "Recall total:  0.7376805893475993\n",
      "F1 total:  0.7370682456398157\n",
      "BACC total:  0.7376805893475993\n",
      "MCC total:  0.9485635273525849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_00.fit(X_train, y_train, eval_set=(X_test, y_test), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat_prob = cat_00.predict_proba(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Catboost base model', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    pred_label = preds_cat\n",
    "    \n",
    "    \n",
    "\n",
    "    # pred_label = y_pred\n",
    "\n",
    "    name = 'cat'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "start = time.time()\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Assuming you have your features and labels as X and y\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb = xgb_00.predict(dtest)\n",
    "# preds_xgb_prob = xgb_00.predict_proba(dtest)\n",
    "\n",
    "\n",
    "# Get class probabilities\n",
    "# Assuming binary classification, get the probability for the positive class (class 1)\n",
    "preds_xgb_margin = xgb_00.predict(dtest, output_margin=True)\n",
    "preds_xgb_prob = 1 / (1 + np.exp(-preds_xgb_margin))\n",
    "\n",
    "# Print or use positive_class_probabilities as needed\n",
    "# print(positive_class_probabilities)\n",
    "\n",
    "\n",
    "# Convert predicted probabilities to class labels (if necessary)\n",
    "# y_pred_labels = [round(value) for value in y_pred]\n",
    "\n",
    "# Evaluate the accuracy\n",
    "# accuracy = accuracy_score(y_test, y_pred)\n",
    "# print(\"Accuracy: %.2f%%\" % (accuracy * 100.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0      1.0     2.0    3.0  4.0\n",
      "0.0  22366.0    267.0   332.0  170.0  0.0\n",
      "1.0    496.0  15408.0   115.0   13.0  0.0\n",
      "2.0    210.0     87.0  3862.0   42.0  0.0\n",
      "3.0    432.0      5.0    27.0  696.0  0.0\n",
      "4.0     14.0      2.0     0.0   12.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9500852859323099\n",
      "Precision total:  0.7129573178306516\n",
      "Recall total:  0.6894286183142595\n",
      "F1 total:  0.6995382651707033\n",
      "BACC total:  0.6894286183142595\n",
      "MCC total:  0.9153569000691623\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    pred_label = preds_xgb\n",
    "    # pred_label = label[ypred]\n",
    "    name = 'xgb'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2     3    4\n",
      "0  21606.0    481.0   987.0  61.0  0.0\n",
      "1   1280.0  14668.0    84.0   0.0  0.0\n",
      "2    939.0   1227.0  2035.0   0.0  0.0\n",
      "3    994.0     38.0   119.0   9.0  0.0\n",
      "4     19.0      5.0     3.0   1.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8599964090133764\n",
      "Precision total:  0.5040827875425867\n",
      "Recall total:  0.468199383046076\n",
      "F1 total:  0.47344821277630506\n",
      "BACC total:  0.468199383046076\n",
      "MCC total:  0.7571187366387974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_00 = LogisticRegression()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start_lr = start = time.time()\n",
    "    logreg_00.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "# if 1 == 1:\n",
    "    # logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_lr = preds_logreg =logreg_00.predict(X_test)\n",
    "    preds_lr_prob = logreg_00.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "    pred_label = preds_logreg\n",
    "    name = 'lr'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    \n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging DT  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3     4\n",
      "0  22761.0    117.0    95.0  151.0  11.0\n",
      "1    140.0  15878.0    11.0    3.0   0.0\n",
      "2    145.0     16.0  4023.0   17.0   0.0\n",
      "3    155.0      1.0    12.0  983.0   9.0\n",
      "4     13.0      0.0     0.0    9.0   6.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9796884819104049\n",
      "Precision total:  0.8039237494979826\n",
      "Recall total:  0.7987113745216188\n",
      "F1 total:  0.8012456019089804\n",
      "BACC total:  0.7987113745216188\n",
      "MCC total:  0.9656207758964955\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "start = time.time()\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "\n",
    "pred_label = y_pred\n",
    "name = 'bag_dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2    3    4\n",
      "0  21595.0    493.0  1043.0  4.0  0.0\n",
      "1   1286.0  14609.0   137.0  0.0  0.0\n",
      "2    629.0   1195.0  2377.0  0.0  0.0\n",
      "3    976.0     59.0   125.0  0.0  0.0\n",
      "4     21.0      4.0     3.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8658990932758776\n",
      "Precision total:  0.48383899131939734\n",
      "Recall total:  0.48209837460833416\n",
      "F1 total:  0.4822812721811104\n",
      "BACC total:  0.48209837460833416\n",
      "MCC total:  0.768152422109659\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "## bagging  with SVM\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3    4\n",
      "0  22460.0    177.0   276.0  220.0  2.0\n",
      "1    164.0  15789.0    59.0   20.0  0.0\n",
      "2    145.0     24.0  3997.0   35.0  0.0\n",
      "3    209.0      3.0    19.0  929.0  0.0\n",
      "4     18.0      2.0     0.0    8.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9690052967052698\n",
      "Precision total:  0.7297907234503216\n",
      "Recall total:  0.7415936888690615\n",
      "F1 total:  0.7355573167145599\n",
      "BACC total:  0.7415936888690615\n",
      "MCC total:  0.947748301022522\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "import time\n",
    "start = time.time()\n",
    "# create MLPClassifier instance\n",
    "mlp_00 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bagging KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3    4\n",
      "0  22582.0    185.0   156.0  207.0  5.0\n",
      "1     99.0  15902.0    19.0   12.0  0.0\n",
      "2    148.0     14.0  4013.0   26.0  0.0\n",
      "3    151.0      0.0    14.0  992.0  3.0\n",
      "4     17.0      2.0     0.0    9.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9760526079540354\n",
      "Precision total:  0.7441282225350755\n",
      "Recall total:  0.7556818409301966\n",
      "F1 total:  0.7496913327100282\n",
      "BACC total:  0.7556818409301966\n",
      "MCC total:  0.959586011927459\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_00=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "base_classifier = knn_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### bag LogRegression\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "start = time.time()\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_00 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2    3    4\n",
      "0  23130.0      9.0    39.0  0.0  0.0\n",
      "1    724.0  15247.0     2.0  0.0  0.0\n",
      "2    274.0     55.0  3879.0  0.0  0.0\n",
      "3   1111.0     16.0    31.0  0.0  0.0\n",
      "4     39.0      0.0     0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9483795672861118\n",
      "Precision total:  0.578316428205879\n",
      "Recall total:  0.5748585920561518\n",
      "F1 total:  0.5759563112544444\n",
      "BACC total:  0.5748585920561518\n",
      "MCC total:  0.9129812471687307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3     4\n",
      "0  21757.0   1056.0   271.0   87.0   7.0\n",
      "1   1738.0  10267.0  3968.0    0.0   0.0\n",
      "2   1509.0    310.0  2389.0    0.0   0.0\n",
      "3    504.0      0.0    46.0  606.0   2.0\n",
      "4     13.0      0.0     0.0   10.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.7863138522309004\n",
      "Precision total:  0.7190130421953926\n",
      "Recall total:  0.616552930067735\n",
      "F1 total:  0.6455379818656247\n",
      "BACC total:  0.616552930067735\n",
      "MCC total:  0.6486334802334476\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2       3     4\n",
      "0  23126.0      7.0    21.0    21.0   3.0\n",
      "1     16.0  15954.0     3.0     0.0   0.0\n",
      "2     22.0      5.0  4180.0     1.0   0.0\n",
      "3     63.0      0.0     1.0  1090.0   4.0\n",
      "4     20.0      0.0     1.0     2.0  16.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9957357033845049\n",
      "Precision total:  0.9323939946356916\n",
      "Recall total:  0.8682894938848295\n",
      "F1 total:  0.8929042317678464\n",
      "BACC total:  0.8682894938848295\n",
      "MCC total:  0.9927829494524889\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging Catboost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.2832155\ttotal: 49.3ms\tremaining: 4.88s\n",
      "1:\tlearn: 1.0881239\ttotal: 79.8ms\tremaining: 3.91s\n",
      "2:\tlearn: 0.9447266\ttotal: 112ms\tremaining: 3.61s\n",
      "3:\tlearn: 0.8356397\ttotal: 141ms\tremaining: 3.39s\n",
      "4:\tlearn: 0.7438959\ttotal: 171ms\tremaining: 3.25s\n",
      "5:\tlearn: 0.6696365\ttotal: 203ms\tremaining: 3.18s\n",
      "6:\tlearn: 0.6058634\ttotal: 231ms\tremaining: 3.07s\n",
      "7:\tlearn: 0.5499808\ttotal: 260ms\tremaining: 2.99s\n",
      "8:\tlearn: 0.5007627\ttotal: 292ms\tremaining: 2.95s\n",
      "9:\tlearn: 0.4598415\ttotal: 319ms\tremaining: 2.87s\n",
      "10:\tlearn: 0.4235366\ttotal: 350ms\tremaining: 2.83s\n",
      "11:\tlearn: 0.3915779\ttotal: 377ms\tremaining: 2.77s\n",
      "12:\tlearn: 0.3618146\ttotal: 408ms\tremaining: 2.73s\n",
      "13:\tlearn: 0.3351684\ttotal: 437ms\tremaining: 2.68s\n",
      "14:\tlearn: 0.3103519\ttotal: 470ms\tremaining: 2.67s\n",
      "15:\tlearn: 0.2896206\ttotal: 496ms\tremaining: 2.6s\n",
      "16:\tlearn: 0.2703089\ttotal: 530ms\tremaining: 2.59s\n",
      "17:\tlearn: 0.2535468\ttotal: 559ms\tremaining: 2.55s\n",
      "18:\tlearn: 0.2388444\ttotal: 589ms\tremaining: 2.51s\n",
      "19:\tlearn: 0.2241902\ttotal: 619ms\tremaining: 2.48s\n",
      "20:\tlearn: 0.2099532\ttotal: 650ms\tremaining: 2.44s\n",
      "21:\tlearn: 0.1978543\ttotal: 681ms\tremaining: 2.41s\n",
      "22:\tlearn: 0.1859443\ttotal: 712ms\tremaining: 2.38s\n",
      "23:\tlearn: 0.1762325\ttotal: 740ms\tremaining: 2.34s\n",
      "24:\tlearn: 0.1671217\ttotal: 771ms\tremaining: 2.31s\n",
      "25:\tlearn: 0.1596902\ttotal: 801ms\tremaining: 2.28s\n",
      "26:\tlearn: 0.1533739\ttotal: 825ms\tremaining: 2.23s\n",
      "27:\tlearn: 0.1468762\ttotal: 852ms\tremaining: 2.19s\n",
      "28:\tlearn: 0.1408693\ttotal: 877ms\tremaining: 2.15s\n",
      "29:\tlearn: 0.1347196\ttotal: 905ms\tremaining: 2.11s\n",
      "30:\tlearn: 0.1288196\ttotal: 935ms\tremaining: 2.08s\n",
      "31:\tlearn: 0.1230463\ttotal: 963ms\tremaining: 2.05s\n",
      "32:\tlearn: 0.1189425\ttotal: 988ms\tremaining: 2s\n",
      "33:\tlearn: 0.1149771\ttotal: 1.01s\tremaining: 1.97s\n",
      "34:\tlearn: 0.1109975\ttotal: 1.04s\tremaining: 1.93s\n",
      "35:\tlearn: 0.1069690\ttotal: 1.07s\tremaining: 1.9s\n",
      "36:\tlearn: 0.1033389\ttotal: 1.09s\tremaining: 1.86s\n",
      "37:\tlearn: 0.1001247\ttotal: 1.12s\tremaining: 1.83s\n",
      "38:\tlearn: 0.0976828\ttotal: 1.14s\tremaining: 1.79s\n",
      "39:\tlearn: 0.0945394\ttotal: 1.17s\tremaining: 1.75s\n",
      "40:\tlearn: 0.0914302\ttotal: 1.2s\tremaining: 1.72s\n",
      "41:\tlearn: 0.0893448\ttotal: 1.22s\tremaining: 1.69s\n",
      "42:\tlearn: 0.0871119\ttotal: 1.25s\tremaining: 1.66s\n",
      "43:\tlearn: 0.0849275\ttotal: 1.28s\tremaining: 1.62s\n",
      "44:\tlearn: 0.0824224\ttotal: 1.3s\tremaining: 1.59s\n",
      "45:\tlearn: 0.0802032\ttotal: 1.33s\tremaining: 1.56s\n",
      "46:\tlearn: 0.0788322\ttotal: 1.35s\tremaining: 1.52s\n",
      "47:\tlearn: 0.0773181\ttotal: 1.38s\tremaining: 1.49s\n",
      "48:\tlearn: 0.0755254\ttotal: 1.4s\tremaining: 1.46s\n",
      "49:\tlearn: 0.0739850\ttotal: 1.43s\tremaining: 1.43s\n",
      "50:\tlearn: 0.0726458\ttotal: 1.45s\tremaining: 1.4s\n",
      "51:\tlearn: 0.0714981\ttotal: 1.47s\tremaining: 1.36s\n",
      "52:\tlearn: 0.0700069\ttotal: 1.5s\tremaining: 1.33s\n",
      "53:\tlearn: 0.0679630\ttotal: 1.53s\tremaining: 1.3s\n",
      "54:\tlearn: 0.0666576\ttotal: 1.55s\tremaining: 1.27s\n",
      "55:\tlearn: 0.0655453\ttotal: 1.58s\tremaining: 1.24s\n",
      "56:\tlearn: 0.0647709\ttotal: 1.6s\tremaining: 1.21s\n",
      "57:\tlearn: 0.0639870\ttotal: 1.62s\tremaining: 1.17s\n",
      "58:\tlearn: 0.0628241\ttotal: 1.65s\tremaining: 1.14s\n",
      "59:\tlearn: 0.0617318\ttotal: 1.67s\tremaining: 1.11s\n",
      "60:\tlearn: 0.0601485\ttotal: 1.7s\tremaining: 1.09s\n",
      "61:\tlearn: 0.0595817\ttotal: 1.72s\tremaining: 1.05s\n",
      "62:\tlearn: 0.0591066\ttotal: 1.74s\tremaining: 1.02s\n",
      "63:\tlearn: 0.0578428\ttotal: 1.77s\tremaining: 993ms\n",
      "64:\tlearn: 0.0575462\ttotal: 1.78s\tremaining: 962ms\n",
      "65:\tlearn: 0.0568574\ttotal: 1.81s\tremaining: 933ms\n",
      "66:\tlearn: 0.0561026\ttotal: 1.84s\tremaining: 904ms\n",
      "67:\tlearn: 0.0554739\ttotal: 1.86s\tremaining: 874ms\n",
      "68:\tlearn: 0.0548164\ttotal: 1.88s\tremaining: 845ms\n",
      "69:\tlearn: 0.0537983\ttotal: 1.91s\tremaining: 818ms\n",
      "70:\tlearn: 0.0525297\ttotal: 1.94s\tremaining: 793ms\n",
      "71:\tlearn: 0.0515766\ttotal: 1.97s\tremaining: 765ms\n",
      "72:\tlearn: 0.0504549\ttotal: 2s\tremaining: 740ms\n",
      "73:\tlearn: 0.0497472\ttotal: 2.03s\tremaining: 713ms\n",
      "74:\tlearn: 0.0491142\ttotal: 2.05s\tremaining: 684ms\n",
      "75:\tlearn: 0.0485689\ttotal: 2.08s\tremaining: 656ms\n",
      "76:\tlearn: 0.0482795\ttotal: 2.1s\tremaining: 626ms\n",
      "77:\tlearn: 0.0476913\ttotal: 2.12s\tremaining: 598ms\n",
      "78:\tlearn: 0.0470821\ttotal: 2.14s\tremaining: 570ms\n",
      "79:\tlearn: 0.0467816\ttotal: 2.16s\tremaining: 541ms\n",
      "80:\tlearn: 0.0465214\ttotal: 2.19s\tremaining: 513ms\n",
      "81:\tlearn: 0.0460096\ttotal: 2.21s\tremaining: 485ms\n",
      "82:\tlearn: 0.0453564\ttotal: 2.23s\tremaining: 458ms\n",
      "83:\tlearn: 0.0450328\ttotal: 2.26s\tremaining: 430ms\n",
      "84:\tlearn: 0.0446397\ttotal: 2.28s\tremaining: 402ms\n",
      "85:\tlearn: 0.0443288\ttotal: 2.3s\tremaining: 375ms\n",
      "86:\tlearn: 0.0440116\ttotal: 2.32s\tremaining: 347ms\n",
      "87:\tlearn: 0.0437656\ttotal: 2.35s\tremaining: 320ms\n",
      "88:\tlearn: 0.0435907\ttotal: 2.37s\tremaining: 293ms\n",
      "89:\tlearn: 0.0432984\ttotal: 2.39s\tremaining: 265ms\n",
      "90:\tlearn: 0.0431396\ttotal: 2.41s\tremaining: 238ms\n",
      "91:\tlearn: 0.0425298\ttotal: 2.43s\tremaining: 212ms\n",
      "92:\tlearn: 0.0421702\ttotal: 2.46s\tremaining: 185ms\n",
      "93:\tlearn: 0.0417927\ttotal: 2.48s\tremaining: 158ms\n",
      "94:\tlearn: 0.0416341\ttotal: 2.5s\tremaining: 131ms\n",
      "95:\tlearn: 0.0412461\ttotal: 2.52s\tremaining: 105ms\n",
      "96:\tlearn: 0.0408529\ttotal: 2.55s\tremaining: 78.7ms\n",
      "97:\tlearn: 0.0403576\ttotal: 2.57s\tremaining: 52.5ms\n",
      "98:\tlearn: 0.0400449\ttotal: 2.59s\tremaining: 26.2ms\n",
      "99:\tlearn: 0.0396714\ttotal: 2.62s\tremaining: 0us\n",
      "0:\tlearn: 1.2823681\ttotal: 41.8ms\tremaining: 4.14s\n",
      "1:\tlearn: 1.0815322\ttotal: 73.5ms\tremaining: 3.6s\n",
      "2:\tlearn: 0.9346001\ttotal: 103ms\tremaining: 3.33s\n",
      "3:\tlearn: 0.8194818\ttotal: 136ms\tremaining: 3.27s\n",
      "4:\tlearn: 0.7279941\ttotal: 167ms\tremaining: 3.17s\n",
      "5:\tlearn: 0.6533265\ttotal: 195ms\tremaining: 3.06s\n",
      "6:\tlearn: 0.5897140\ttotal: 229ms\tremaining: 3.04s\n",
      "7:\tlearn: 0.5350698\ttotal: 260ms\tremaining: 2.99s\n",
      "8:\tlearn: 0.4872659\ttotal: 292ms\tremaining: 2.95s\n",
      "9:\tlearn: 0.4475888\ttotal: 320ms\tremaining: 2.88s\n",
      "10:\tlearn: 0.4110695\ttotal: 349ms\tremaining: 2.82s\n",
      "11:\tlearn: 0.3797083\ttotal: 378ms\tremaining: 2.77s\n",
      "12:\tlearn: 0.3512678\ttotal: 412ms\tremaining: 2.75s\n",
      "13:\tlearn: 0.3263982\ttotal: 436ms\tremaining: 2.68s\n",
      "14:\tlearn: 0.3009146\ttotal: 469ms\tremaining: 2.66s\n",
      "15:\tlearn: 0.2804994\ttotal: 496ms\tremaining: 2.6s\n",
      "16:\tlearn: 0.2613995\ttotal: 528ms\tremaining: 2.58s\n",
      "17:\tlearn: 0.2444719\ttotal: 559ms\tremaining: 2.55s\n",
      "18:\tlearn: 0.2291891\ttotal: 591ms\tremaining: 2.52s\n",
      "19:\tlearn: 0.2156545\ttotal: 621ms\tremaining: 2.48s\n",
      "20:\tlearn: 0.2027745\ttotal: 653ms\tremaining: 2.46s\n",
      "21:\tlearn: 0.1910516\ttotal: 683ms\tremaining: 2.42s\n",
      "22:\tlearn: 0.1818934\ttotal: 707ms\tremaining: 2.37s\n",
      "23:\tlearn: 0.1730321\ttotal: 732ms\tremaining: 2.32s\n",
      "24:\tlearn: 0.1636911\ttotal: 759ms\tremaining: 2.28s\n",
      "25:\tlearn: 0.1554507\ttotal: 787ms\tremaining: 2.24s\n",
      "26:\tlearn: 0.1489867\ttotal: 815ms\tremaining: 2.2s\n",
      "27:\tlearn: 0.1431632\ttotal: 842ms\tremaining: 2.16s\n",
      "28:\tlearn: 0.1371152\ttotal: 869ms\tremaining: 2.13s\n",
      "29:\tlearn: 0.1322832\ttotal: 897ms\tremaining: 2.09s\n",
      "30:\tlearn: 0.1259233\ttotal: 927ms\tremaining: 2.06s\n",
      "31:\tlearn: 0.1209594\ttotal: 954ms\tremaining: 2.03s\n",
      "32:\tlearn: 0.1159773\ttotal: 988ms\tremaining: 2.01s\n",
      "33:\tlearn: 0.1117007\ttotal: 1.02s\tremaining: 1.97s\n",
      "34:\tlearn: 0.1076741\ttotal: 1.04s\tremaining: 1.94s\n",
      "35:\tlearn: 0.1041363\ttotal: 1.07s\tremaining: 1.91s\n",
      "36:\tlearn: 0.1012456\ttotal: 1.09s\tremaining: 1.86s\n",
      "37:\tlearn: 0.0985470\ttotal: 1.12s\tremaining: 1.83s\n",
      "38:\tlearn: 0.0967082\ttotal: 1.14s\tremaining: 1.78s\n",
      "39:\tlearn: 0.0940140\ttotal: 1.16s\tremaining: 1.75s\n",
      "40:\tlearn: 0.0914168\ttotal: 1.19s\tremaining: 1.71s\n",
      "41:\tlearn: 0.0891491\ttotal: 1.22s\tremaining: 1.68s\n",
      "42:\tlearn: 0.0863417\ttotal: 1.24s\tremaining: 1.64s\n",
      "43:\tlearn: 0.0844037\ttotal: 1.26s\tremaining: 1.6s\n",
      "44:\tlearn: 0.0827645\ttotal: 1.28s\tremaining: 1.57s\n",
      "45:\tlearn: 0.0812612\ttotal: 1.31s\tremaining: 1.54s\n",
      "46:\tlearn: 0.0794022\ttotal: 1.33s\tremaining: 1.5s\n",
      "47:\tlearn: 0.0775023\ttotal: 1.36s\tremaining: 1.47s\n",
      "48:\tlearn: 0.0757059\ttotal: 1.38s\tremaining: 1.44s\n",
      "49:\tlearn: 0.0741588\ttotal: 1.4s\tremaining: 1.4s\n",
      "50:\tlearn: 0.0727830\ttotal: 1.43s\tremaining: 1.37s\n",
      "51:\tlearn: 0.0715808\ttotal: 1.45s\tremaining: 1.34s\n",
      "52:\tlearn: 0.0698408\ttotal: 1.48s\tremaining: 1.31s\n",
      "53:\tlearn: 0.0686963\ttotal: 1.5s\tremaining: 1.28s\n",
      "54:\tlearn: 0.0680451\ttotal: 1.52s\tremaining: 1.25s\n",
      "55:\tlearn: 0.0662301\ttotal: 1.55s\tremaining: 1.22s\n",
      "56:\tlearn: 0.0646837\ttotal: 1.57s\tremaining: 1.19s\n",
      "57:\tlearn: 0.0629182\ttotal: 1.6s\tremaining: 1.16s\n",
      "58:\tlearn: 0.0622836\ttotal: 1.62s\tremaining: 1.12s\n",
      "59:\tlearn: 0.0616541\ttotal: 1.64s\tremaining: 1.09s\n",
      "60:\tlearn: 0.0604014\ttotal: 1.67s\tremaining: 1.06s\n",
      "61:\tlearn: 0.0596972\ttotal: 1.69s\tremaining: 1.03s\n",
      "62:\tlearn: 0.0584425\ttotal: 1.71s\tremaining: 1.01s\n",
      "63:\tlearn: 0.0577403\ttotal: 1.74s\tremaining: 976ms\n",
      "64:\tlearn: 0.0569525\ttotal: 1.76s\tremaining: 948ms\n",
      "65:\tlearn: 0.0565053\ttotal: 1.78s\tremaining: 918ms\n",
      "66:\tlearn: 0.0554980\ttotal: 1.81s\tremaining: 891ms\n",
      "67:\tlearn: 0.0549933\ttotal: 1.83s\tremaining: 861ms\n",
      "68:\tlearn: 0.0545275\ttotal: 1.85s\tremaining: 831ms\n",
      "69:\tlearn: 0.0540942\ttotal: 1.87s\tremaining: 802ms\n",
      "70:\tlearn: 0.0536171\ttotal: 1.89s\tremaining: 773ms\n",
      "71:\tlearn: 0.0528683\ttotal: 1.92s\tremaining: 746ms\n",
      "72:\tlearn: 0.0520898\ttotal: 1.95s\tremaining: 719ms\n",
      "73:\tlearn: 0.0513187\ttotal: 1.97s\tremaining: 693ms\n",
      "74:\tlearn: 0.0507879\ttotal: 1.99s\tremaining: 665ms\n",
      "75:\tlearn: 0.0503007\ttotal: 2.02s\tremaining: 638ms\n",
      "76:\tlearn: 0.0498692\ttotal: 2.04s\tremaining: 610ms\n",
      "77:\tlearn: 0.0494350\ttotal: 2.06s\tremaining: 583ms\n",
      "78:\tlearn: 0.0488795\ttotal: 2.09s\tremaining: 555ms\n",
      "79:\tlearn: 0.0486508\ttotal: 2.11s\tremaining: 528ms\n",
      "80:\tlearn: 0.0481091\ttotal: 2.13s\tremaining: 501ms\n",
      "81:\tlearn: 0.0473971\ttotal: 2.16s\tremaining: 474ms\n",
      "82:\tlearn: 0.0470128\ttotal: 2.18s\tremaining: 447ms\n",
      "83:\tlearn: 0.0463455\ttotal: 2.21s\tremaining: 421ms\n",
      "84:\tlearn: 0.0462564\ttotal: 2.23s\tremaining: 393ms\n",
      "85:\tlearn: 0.0461042\ttotal: 2.25s\tremaining: 366ms\n",
      "86:\tlearn: 0.0455034\ttotal: 2.27s\tremaining: 340ms\n",
      "87:\tlearn: 0.0452074\ttotal: 2.3s\tremaining: 313ms\n",
      "88:\tlearn: 0.0446630\ttotal: 2.32s\tremaining: 287ms\n",
      "89:\tlearn: 0.0442762\ttotal: 2.35s\tremaining: 261ms\n",
      "90:\tlearn: 0.0439216\ttotal: 2.37s\tremaining: 234ms\n",
      "91:\tlearn: 0.0436812\ttotal: 2.39s\tremaining: 208ms\n",
      "92:\tlearn: 0.0435596\ttotal: 2.41s\tremaining: 181ms\n",
      "93:\tlearn: 0.0430055\ttotal: 2.43s\tremaining: 155ms\n",
      "94:\tlearn: 0.0428808\ttotal: 2.46s\tremaining: 129ms\n",
      "95:\tlearn: 0.0424943\ttotal: 2.48s\tremaining: 103ms\n",
      "96:\tlearn: 0.0420313\ttotal: 2.5s\tremaining: 77.4ms\n",
      "97:\tlearn: 0.0417353\ttotal: 2.52s\tremaining: 51.5ms\n",
      "98:\tlearn: 0.0414223\ttotal: 2.54s\tremaining: 25.7ms\n",
      "99:\tlearn: 0.0411344\ttotal: 2.57s\tremaining: 0us\n",
      "0:\tlearn: 1.2967849\ttotal: 36.3ms\tremaining: 3.59s\n",
      "1:\tlearn: 1.1011068\ttotal: 66.2ms\tremaining: 3.25s\n",
      "2:\tlearn: 0.9500989\ttotal: 96.3ms\tremaining: 3.11s\n",
      "3:\tlearn: 0.8322857\ttotal: 130ms\tremaining: 3.12s\n",
      "4:\tlearn: 0.7386643\ttotal: 159ms\tremaining: 3.02s\n",
      "5:\tlearn: 0.6629101\ttotal: 191ms\tremaining: 2.99s\n",
      "6:\tlearn: 0.5982774\ttotal: 223ms\tremaining: 2.96s\n",
      "7:\tlearn: 0.5434819\ttotal: 253ms\tremaining: 2.91s\n",
      "8:\tlearn: 0.4959254\ttotal: 282ms\tremaining: 2.85s\n",
      "9:\tlearn: 0.4546682\ttotal: 312ms\tremaining: 2.81s\n",
      "10:\tlearn: 0.4174210\ttotal: 345ms\tremaining: 2.79s\n",
      "11:\tlearn: 0.3862689\ttotal: 370ms\tremaining: 2.72s\n",
      "12:\tlearn: 0.3561623\ttotal: 401ms\tremaining: 2.69s\n",
      "13:\tlearn: 0.3313025\ttotal: 427ms\tremaining: 2.62s\n",
      "14:\tlearn: 0.3060093\ttotal: 458ms\tremaining: 2.59s\n",
      "15:\tlearn: 0.2846582\ttotal: 491ms\tremaining: 2.58s\n",
      "16:\tlearn: 0.2658063\ttotal: 519ms\tremaining: 2.53s\n",
      "17:\tlearn: 0.2488860\ttotal: 549ms\tremaining: 2.5s\n",
      "18:\tlearn: 0.2341963\ttotal: 577ms\tremaining: 2.46s\n",
      "19:\tlearn: 0.2199211\ttotal: 605ms\tremaining: 2.42s\n",
      "20:\tlearn: 0.2089953\ttotal: 633ms\tremaining: 2.38s\n",
      "21:\tlearn: 0.1973005\ttotal: 663ms\tremaining: 2.35s\n",
      "22:\tlearn: 0.1871978\ttotal: 689ms\tremaining: 2.31s\n",
      "23:\tlearn: 0.1767454\ttotal: 717ms\tremaining: 2.27s\n",
      "24:\tlearn: 0.1678119\ttotal: 745ms\tremaining: 2.24s\n",
      "25:\tlearn: 0.1594207\ttotal: 771ms\tremaining: 2.19s\n",
      "26:\tlearn: 0.1534592\ttotal: 797ms\tremaining: 2.15s\n",
      "27:\tlearn: 0.1458749\ttotal: 827ms\tremaining: 2.13s\n",
      "28:\tlearn: 0.1407025\ttotal: 852ms\tremaining: 2.09s\n",
      "29:\tlearn: 0.1343169\ttotal: 879ms\tremaining: 2.05s\n",
      "30:\tlearn: 0.1283709\ttotal: 907ms\tremaining: 2.02s\n",
      "31:\tlearn: 0.1228400\ttotal: 934ms\tremaining: 1.98s\n",
      "32:\tlearn: 0.1180935\ttotal: 960ms\tremaining: 1.95s\n",
      "33:\tlearn: 0.1137244\ttotal: 988ms\tremaining: 1.92s\n",
      "34:\tlearn: 0.1097406\ttotal: 1.01s\tremaining: 1.88s\n",
      "35:\tlearn: 0.1056134\ttotal: 1.04s\tremaining: 1.85s\n",
      "36:\tlearn: 0.1018460\ttotal: 1.07s\tremaining: 1.82s\n",
      "37:\tlearn: 0.0994274\ttotal: 1.09s\tremaining: 1.78s\n",
      "38:\tlearn: 0.0958730\ttotal: 1.12s\tremaining: 1.76s\n",
      "39:\tlearn: 0.0930286\ttotal: 1.15s\tremaining: 1.72s\n",
      "40:\tlearn: 0.0909457\ttotal: 1.17s\tremaining: 1.69s\n",
      "41:\tlearn: 0.0885066\ttotal: 1.2s\tremaining: 1.66s\n",
      "42:\tlearn: 0.0860347\ttotal: 1.23s\tremaining: 1.62s\n",
      "43:\tlearn: 0.0840986\ttotal: 1.25s\tremaining: 1.59s\n",
      "44:\tlearn: 0.0816645\ttotal: 1.28s\tremaining: 1.56s\n",
      "45:\tlearn: 0.0798896\ttotal: 1.3s\tremaining: 1.52s\n",
      "46:\tlearn: 0.0780872\ttotal: 1.32s\tremaining: 1.49s\n",
      "47:\tlearn: 0.0766158\ttotal: 1.35s\tremaining: 1.46s\n",
      "48:\tlearn: 0.0749855\ttotal: 1.37s\tremaining: 1.43s\n",
      "49:\tlearn: 0.0725295\ttotal: 1.4s\tremaining: 1.4s\n",
      "50:\tlearn: 0.0713501\ttotal: 1.43s\tremaining: 1.37s\n",
      "51:\tlearn: 0.0700272\ttotal: 1.45s\tremaining: 1.34s\n",
      "52:\tlearn: 0.0688610\ttotal: 1.47s\tremaining: 1.3s\n",
      "53:\tlearn: 0.0679180\ttotal: 1.49s\tremaining: 1.27s\n",
      "54:\tlearn: 0.0667068\ttotal: 1.52s\tremaining: 1.24s\n",
      "55:\tlearn: 0.0650768\ttotal: 1.54s\tremaining: 1.21s\n",
      "56:\tlearn: 0.0638728\ttotal: 1.56s\tremaining: 1.18s\n",
      "57:\tlearn: 0.0626965\ttotal: 1.59s\tremaining: 1.15s\n",
      "58:\tlearn: 0.0613879\ttotal: 1.62s\tremaining: 1.12s\n",
      "59:\tlearn: 0.0599645\ttotal: 1.64s\tremaining: 1.09s\n",
      "60:\tlearn: 0.0585882\ttotal: 1.67s\tremaining: 1.07s\n",
      "61:\tlearn: 0.0581111\ttotal: 1.69s\tremaining: 1.03s\n",
      "62:\tlearn: 0.0569197\ttotal: 1.71s\tremaining: 1.01s\n",
      "63:\tlearn: 0.0558214\ttotal: 1.74s\tremaining: 979ms\n",
      "64:\tlearn: 0.0551876\ttotal: 1.76s\tremaining: 950ms\n",
      "65:\tlearn: 0.0547873\ttotal: 1.78s\tremaining: 919ms\n",
      "66:\tlearn: 0.0540643\ttotal: 1.81s\tremaining: 892ms\n",
      "67:\tlearn: 0.0537402\ttotal: 1.83s\tremaining: 861ms\n",
      "68:\tlearn: 0.0529052\ttotal: 1.86s\tremaining: 834ms\n",
      "69:\tlearn: 0.0521797\ttotal: 1.88s\tremaining: 805ms\n",
      "70:\tlearn: 0.0514896\ttotal: 1.9s\tremaining: 777ms\n",
      "71:\tlearn: 0.0510488\ttotal: 1.92s\tremaining: 748ms\n",
      "72:\tlearn: 0.0505750\ttotal: 1.95s\tremaining: 720ms\n",
      "73:\tlearn: 0.0502201\ttotal: 1.97s\tremaining: 691ms\n",
      "74:\tlearn: 0.0496719\ttotal: 1.99s\tremaining: 662ms\n",
      "75:\tlearn: 0.0494692\ttotal: 2.01s\tremaining: 634ms\n",
      "76:\tlearn: 0.0487709\ttotal: 2.03s\tremaining: 607ms\n",
      "77:\tlearn: 0.0484749\ttotal: 2.05s\tremaining: 579ms\n",
      "78:\tlearn: 0.0477978\ttotal: 2.08s\tremaining: 553ms\n",
      "79:\tlearn: 0.0474665\ttotal: 2.1s\tremaining: 526ms\n",
      "80:\tlearn: 0.0469555\ttotal: 2.13s\tremaining: 499ms\n",
      "81:\tlearn: 0.0466483\ttotal: 2.15s\tremaining: 471ms\n",
      "82:\tlearn: 0.0457998\ttotal: 2.17s\tremaining: 445ms\n",
      "83:\tlearn: 0.0451159\ttotal: 2.2s\tremaining: 419ms\n",
      "84:\tlearn: 0.0449698\ttotal: 2.22s\tremaining: 392ms\n",
      "85:\tlearn: 0.0446621\ttotal: 2.24s\tremaining: 365ms\n",
      "86:\tlearn: 0.0443359\ttotal: 2.26s\tremaining: 338ms\n",
      "87:\tlearn: 0.0439056\ttotal: 2.29s\tremaining: 312ms\n",
      "88:\tlearn: 0.0436958\ttotal: 2.31s\tremaining: 285ms\n",
      "89:\tlearn: 0.0431617\ttotal: 2.33s\tremaining: 259ms\n",
      "90:\tlearn: 0.0430333\ttotal: 2.35s\tremaining: 233ms\n",
      "91:\tlearn: 0.0424539\ttotal: 2.38s\tremaining: 207ms\n",
      "92:\tlearn: 0.0418315\ttotal: 2.41s\tremaining: 181ms\n",
      "93:\tlearn: 0.0410992\ttotal: 2.43s\tremaining: 155ms\n",
      "94:\tlearn: 0.0407963\ttotal: 2.46s\tremaining: 129ms\n",
      "95:\tlearn: 0.0403668\ttotal: 2.48s\tremaining: 104ms\n",
      "96:\tlearn: 0.0398728\ttotal: 2.51s\tremaining: 77.7ms\n",
      "97:\tlearn: 0.0394972\ttotal: 2.53s\tremaining: 51.7ms\n",
      "98:\tlearn: 0.0391663\ttotal: 2.55s\tremaining: 25.8ms\n",
      "99:\tlearn: 0.0389128\ttotal: 2.58s\tremaining: 0us\n",
      "0:\tlearn: 1.2827041\ttotal: 39.3ms\tremaining: 3.89s\n",
      "1:\tlearn: 1.0870341\ttotal: 69.9ms\tremaining: 3.42s\n",
      "2:\tlearn: 0.9390760\ttotal: 99.4ms\tremaining: 3.21s\n",
      "3:\tlearn: 0.8261158\ttotal: 129ms\tremaining: 3.1s\n",
      "4:\tlearn: 0.7354807\ttotal: 158ms\tremaining: 3s\n",
      "5:\tlearn: 0.6602329\ttotal: 190ms\tremaining: 2.98s\n",
      "6:\tlearn: 0.5948650\ttotal: 222ms\tremaining: 2.95s\n",
      "7:\tlearn: 0.5388109\ttotal: 253ms\tremaining: 2.9s\n",
      "8:\tlearn: 0.4914985\ttotal: 283ms\tremaining: 2.86s\n",
      "9:\tlearn: 0.4507036\ttotal: 311ms\tremaining: 2.8s\n",
      "10:\tlearn: 0.4150340\ttotal: 340ms\tremaining: 2.75s\n",
      "11:\tlearn: 0.3830775\ttotal: 368ms\tremaining: 2.7s\n",
      "12:\tlearn: 0.3552903\ttotal: 394ms\tremaining: 2.64s\n",
      "13:\tlearn: 0.3299725\ttotal: 420ms\tremaining: 2.58s\n",
      "14:\tlearn: 0.3040437\ttotal: 454ms\tremaining: 2.57s\n",
      "15:\tlearn: 0.2833377\ttotal: 482ms\tremaining: 2.53s\n",
      "16:\tlearn: 0.2647614\ttotal: 514ms\tremaining: 2.51s\n",
      "17:\tlearn: 0.2481603\ttotal: 544ms\tremaining: 2.48s\n",
      "18:\tlearn: 0.2330696\ttotal: 573ms\tremaining: 2.44s\n",
      "19:\tlearn: 0.2176260\ttotal: 604ms\tremaining: 2.41s\n",
      "20:\tlearn: 0.2068435\ttotal: 633ms\tremaining: 2.38s\n",
      "21:\tlearn: 0.1952272\ttotal: 663ms\tremaining: 2.35s\n",
      "22:\tlearn: 0.1840973\ttotal: 694ms\tremaining: 2.32s\n",
      "23:\tlearn: 0.1753580\ttotal: 722ms\tremaining: 2.29s\n",
      "24:\tlearn: 0.1671574\ttotal: 750ms\tremaining: 2.25s\n",
      "25:\tlearn: 0.1596570\ttotal: 777ms\tremaining: 2.21s\n",
      "26:\tlearn: 0.1534893\ttotal: 801ms\tremaining: 2.17s\n",
      "27:\tlearn: 0.1467387\ttotal: 830ms\tremaining: 2.13s\n",
      "28:\tlearn: 0.1400465\ttotal: 857ms\tremaining: 2.1s\n",
      "29:\tlearn: 0.1336776\ttotal: 886ms\tremaining: 2.07s\n",
      "30:\tlearn: 0.1285768\ttotal: 915ms\tremaining: 2.04s\n",
      "31:\tlearn: 0.1232348\ttotal: 943ms\tremaining: 2s\n",
      "32:\tlearn: 0.1194140\ttotal: 964ms\tremaining: 1.96s\n",
      "33:\tlearn: 0.1148337\ttotal: 991ms\tremaining: 1.92s\n",
      "34:\tlearn: 0.1117076\ttotal: 1.01s\tremaining: 1.88s\n",
      "35:\tlearn: 0.1084932\ttotal: 1.04s\tremaining: 1.85s\n",
      "36:\tlearn: 0.1051067\ttotal: 1.06s\tremaining: 1.81s\n",
      "37:\tlearn: 0.1015280\ttotal: 1.09s\tremaining: 1.78s\n",
      "38:\tlearn: 0.0981394\ttotal: 1.12s\tremaining: 1.75s\n",
      "39:\tlearn: 0.0950588\ttotal: 1.14s\tremaining: 1.72s\n",
      "40:\tlearn: 0.0919503\ttotal: 1.17s\tremaining: 1.69s\n",
      "41:\tlearn: 0.0892375\ttotal: 1.2s\tremaining: 1.65s\n",
      "42:\tlearn: 0.0872403\ttotal: 1.22s\tremaining: 1.62s\n",
      "43:\tlearn: 0.0844574\ttotal: 1.25s\tremaining: 1.59s\n",
      "44:\tlearn: 0.0819519\ttotal: 1.27s\tremaining: 1.56s\n",
      "45:\tlearn: 0.0804892\ttotal: 1.3s\tremaining: 1.52s\n",
      "46:\tlearn: 0.0784072\ttotal: 1.32s\tremaining: 1.49s\n",
      "47:\tlearn: 0.0766924\ttotal: 1.35s\tremaining: 1.46s\n",
      "48:\tlearn: 0.0750425\ttotal: 1.37s\tremaining: 1.43s\n",
      "49:\tlearn: 0.0729566\ttotal: 1.4s\tremaining: 1.4s\n",
      "50:\tlearn: 0.0710280\ttotal: 1.42s\tremaining: 1.37s\n",
      "51:\tlearn: 0.0692717\ttotal: 1.46s\tremaining: 1.34s\n",
      "52:\tlearn: 0.0680289\ttotal: 1.48s\tremaining: 1.31s\n",
      "53:\tlearn: 0.0667419\ttotal: 1.5s\tremaining: 1.28s\n",
      "54:\tlearn: 0.0658497\ttotal: 1.52s\tremaining: 1.25s\n",
      "55:\tlearn: 0.0647953\ttotal: 1.55s\tremaining: 1.22s\n",
      "56:\tlearn: 0.0636627\ttotal: 1.57s\tremaining: 1.19s\n",
      "57:\tlearn: 0.0626281\ttotal: 1.6s\tremaining: 1.16s\n",
      "58:\tlearn: 0.0620069\ttotal: 1.62s\tremaining: 1.12s\n",
      "59:\tlearn: 0.0609944\ttotal: 1.64s\tremaining: 1.09s\n",
      "60:\tlearn: 0.0596670\ttotal: 1.66s\tremaining: 1.06s\n",
      "61:\tlearn: 0.0591296\ttotal: 1.68s\tremaining: 1.03s\n",
      "62:\tlearn: 0.0583461\ttotal: 1.7s\tremaining: 1s\n",
      "63:\tlearn: 0.0576209\ttotal: 1.73s\tremaining: 972ms\n",
      "64:\tlearn: 0.0566859\ttotal: 1.75s\tremaining: 944ms\n",
      "65:\tlearn: 0.0556112\ttotal: 1.78s\tremaining: 917ms\n",
      "66:\tlearn: 0.0548269\ttotal: 1.8s\tremaining: 889ms\n",
      "67:\tlearn: 0.0534656\ttotal: 1.84s\tremaining: 864ms\n",
      "68:\tlearn: 0.0521818\ttotal: 1.87s\tremaining: 839ms\n",
      "69:\tlearn: 0.0513319\ttotal: 1.89s\tremaining: 812ms\n",
      "70:\tlearn: 0.0509903\ttotal: 1.92s\tremaining: 782ms\n",
      "71:\tlearn: 0.0499846\ttotal: 1.94s\tremaining: 755ms\n",
      "72:\tlearn: 0.0495400\ttotal: 1.96s\tremaining: 726ms\n",
      "73:\tlearn: 0.0490011\ttotal: 1.99s\tremaining: 698ms\n",
      "74:\tlearn: 0.0485920\ttotal: 2.01s\tremaining: 670ms\n",
      "75:\tlearn: 0.0479912\ttotal: 2.03s\tremaining: 642ms\n",
      "76:\tlearn: 0.0472335\ttotal: 2.06s\tremaining: 614ms\n",
      "77:\tlearn: 0.0468065\ttotal: 2.08s\tremaining: 587ms\n",
      "78:\tlearn: 0.0463728\ttotal: 2.1s\tremaining: 559ms\n",
      "79:\tlearn: 0.0456293\ttotal: 2.13s\tremaining: 532ms\n",
      "80:\tlearn: 0.0454350\ttotal: 2.15s\tremaining: 504ms\n",
      "81:\tlearn: 0.0449384\ttotal: 2.17s\tremaining: 477ms\n",
      "82:\tlearn: 0.0446935\ttotal: 2.19s\tremaining: 449ms\n",
      "83:\tlearn: 0.0442591\ttotal: 2.22s\tremaining: 422ms\n",
      "84:\tlearn: 0.0438200\ttotal: 2.24s\tremaining: 395ms\n",
      "85:\tlearn: 0.0435022\ttotal: 2.26s\tremaining: 368ms\n",
      "86:\tlearn: 0.0433085\ttotal: 2.28s\tremaining: 341ms\n",
      "87:\tlearn: 0.0426705\ttotal: 2.31s\tremaining: 314ms\n",
      "88:\tlearn: 0.0422813\ttotal: 2.33s\tremaining: 288ms\n",
      "89:\tlearn: 0.0420749\ttotal: 2.35s\tremaining: 261ms\n",
      "90:\tlearn: 0.0418921\ttotal: 2.37s\tremaining: 234ms\n",
      "91:\tlearn: 0.0416449\ttotal: 2.39s\tremaining: 208ms\n",
      "92:\tlearn: 0.0413320\ttotal: 2.41s\tremaining: 181ms\n",
      "93:\tlearn: 0.0408458\ttotal: 2.44s\tremaining: 155ms\n",
      "94:\tlearn: 0.0404307\ttotal: 2.46s\tremaining: 130ms\n",
      "95:\tlearn: 0.0399821\ttotal: 2.48s\tremaining: 104ms\n",
      "96:\tlearn: 0.0394940\ttotal: 2.51s\tremaining: 77.6ms\n",
      "97:\tlearn: 0.0392674\ttotal: 2.53s\tremaining: 51.7ms\n",
      "98:\tlearn: 0.0388997\ttotal: 2.56s\tremaining: 25.8ms\n",
      "99:\tlearn: 0.0387880\ttotal: 2.58s\tremaining: 0us\n",
      "0:\tlearn: 1.2826692\ttotal: 41.7ms\tremaining: 4.13s\n",
      "1:\tlearn: 1.0870200\ttotal: 72.5ms\tremaining: 3.55s\n",
      "2:\tlearn: 0.9413644\ttotal: 108ms\tremaining: 3.5s\n",
      "3:\tlearn: 0.8255936\ttotal: 140ms\tremaining: 3.35s\n",
      "4:\tlearn: 0.7371321\ttotal: 170ms\tremaining: 3.22s\n",
      "5:\tlearn: 0.6615839\ttotal: 198ms\tremaining: 3.1s\n",
      "6:\tlearn: 0.5954051\ttotal: 230ms\tremaining: 3.05s\n",
      "7:\tlearn: 0.5394698\ttotal: 258ms\tremaining: 2.96s\n",
      "8:\tlearn: 0.4915811\ttotal: 292ms\tremaining: 2.95s\n",
      "9:\tlearn: 0.4521230\ttotal: 318ms\tremaining: 2.86s\n",
      "10:\tlearn: 0.4147174\ttotal: 346ms\tremaining: 2.8s\n",
      "11:\tlearn: 0.3829784\ttotal: 376ms\tremaining: 2.75s\n",
      "12:\tlearn: 0.3534253\ttotal: 406ms\tremaining: 2.71s\n",
      "13:\tlearn: 0.3275576\ttotal: 435ms\tremaining: 2.67s\n",
      "14:\tlearn: 0.3031379\ttotal: 467ms\tremaining: 2.65s\n",
      "15:\tlearn: 0.2829919\ttotal: 492ms\tremaining: 2.58s\n",
      "16:\tlearn: 0.2644729\ttotal: 523ms\tremaining: 2.55s\n",
      "17:\tlearn: 0.2473438\ttotal: 554ms\tremaining: 2.52s\n",
      "18:\tlearn: 0.2300025\ttotal: 585ms\tremaining: 2.49s\n",
      "19:\tlearn: 0.2157486\ttotal: 615ms\tremaining: 2.46s\n",
      "20:\tlearn: 0.2044365\ttotal: 644ms\tremaining: 2.42s\n",
      "21:\tlearn: 0.1937678\ttotal: 673ms\tremaining: 2.38s\n",
      "22:\tlearn: 0.1837296\ttotal: 704ms\tremaining: 2.36s\n",
      "23:\tlearn: 0.1738713\ttotal: 730ms\tremaining: 2.31s\n",
      "24:\tlearn: 0.1654784\ttotal: 758ms\tremaining: 2.27s\n",
      "25:\tlearn: 0.1576284\ttotal: 785ms\tremaining: 2.23s\n",
      "26:\tlearn: 0.1513954\ttotal: 809ms\tremaining: 2.19s\n",
      "27:\tlearn: 0.1446287\ttotal: 836ms\tremaining: 2.15s\n",
      "28:\tlearn: 0.1394905\ttotal: 861ms\tremaining: 2.11s\n",
      "29:\tlearn: 0.1341357\ttotal: 887ms\tremaining: 2.07s\n",
      "30:\tlearn: 0.1284645\ttotal: 917ms\tremaining: 2.04s\n",
      "31:\tlearn: 0.1236512\ttotal: 944ms\tremaining: 2s\n",
      "32:\tlearn: 0.1194585\ttotal: 969ms\tremaining: 1.97s\n",
      "33:\tlearn: 0.1146152\ttotal: 1s\tremaining: 1.94s\n",
      "34:\tlearn: 0.1107305\ttotal: 1.03s\tremaining: 1.91s\n",
      "35:\tlearn: 0.1067902\ttotal: 1.05s\tremaining: 1.87s\n",
      "36:\tlearn: 0.1031193\ttotal: 1.08s\tremaining: 1.83s\n",
      "37:\tlearn: 0.0997894\ttotal: 1.1s\tremaining: 1.8s\n",
      "38:\tlearn: 0.0962761\ttotal: 1.13s\tremaining: 1.77s\n",
      "39:\tlearn: 0.0939054\ttotal: 1.15s\tremaining: 1.73s\n",
      "40:\tlearn: 0.0909989\ttotal: 1.18s\tremaining: 1.7s\n",
      "41:\tlearn: 0.0879079\ttotal: 1.22s\tremaining: 1.68s\n",
      "42:\tlearn: 0.0861697\ttotal: 1.24s\tremaining: 1.65s\n",
      "43:\tlearn: 0.0834623\ttotal: 1.27s\tremaining: 1.61s\n",
      "44:\tlearn: 0.0817494\ttotal: 1.29s\tremaining: 1.57s\n",
      "45:\tlearn: 0.0797362\ttotal: 1.31s\tremaining: 1.54s\n",
      "46:\tlearn: 0.0778162\ttotal: 1.34s\tremaining: 1.51s\n",
      "47:\tlearn: 0.0758283\ttotal: 1.37s\tremaining: 1.48s\n",
      "48:\tlearn: 0.0740916\ttotal: 1.39s\tremaining: 1.45s\n",
      "49:\tlearn: 0.0721825\ttotal: 1.42s\tremaining: 1.42s\n",
      "50:\tlearn: 0.0703119\ttotal: 1.44s\tremaining: 1.39s\n",
      "51:\tlearn: 0.0686393\ttotal: 1.47s\tremaining: 1.36s\n",
      "52:\tlearn: 0.0675014\ttotal: 1.49s\tremaining: 1.32s\n",
      "53:\tlearn: 0.0663961\ttotal: 1.51s\tremaining: 1.29s\n",
      "54:\tlearn: 0.0652782\ttotal: 1.54s\tremaining: 1.26s\n",
      "55:\tlearn: 0.0636671\ttotal: 1.56s\tremaining: 1.23s\n",
      "56:\tlearn: 0.0625745\ttotal: 1.59s\tremaining: 1.2s\n",
      "57:\tlearn: 0.0612856\ttotal: 1.61s\tremaining: 1.17s\n",
      "58:\tlearn: 0.0604611\ttotal: 1.64s\tremaining: 1.14s\n",
      "59:\tlearn: 0.0593012\ttotal: 1.66s\tremaining: 1.1s\n",
      "60:\tlearn: 0.0586414\ttotal: 1.68s\tremaining: 1.07s\n",
      "61:\tlearn: 0.0581772\ttotal: 1.7s\tremaining: 1.04s\n",
      "62:\tlearn: 0.0571083\ttotal: 1.73s\tremaining: 1.01s\n",
      "63:\tlearn: 0.0564259\ttotal: 1.75s\tremaining: 986ms\n",
      "64:\tlearn: 0.0557845\ttotal: 1.77s\tremaining: 956ms\n",
      "65:\tlearn: 0.0549063\ttotal: 1.8s\tremaining: 928ms\n",
      "66:\tlearn: 0.0541667\ttotal: 1.82s\tremaining: 899ms\n",
      "67:\tlearn: 0.0532038\ttotal: 1.85s\tremaining: 870ms\n",
      "68:\tlearn: 0.0524960\ttotal: 1.87s\tremaining: 841ms\n",
      "69:\tlearn: 0.0515406\ttotal: 1.9s\tremaining: 814ms\n",
      "70:\tlearn: 0.0509320\ttotal: 1.92s\tremaining: 784ms\n",
      "71:\tlearn: 0.0505256\ttotal: 1.94s\tremaining: 755ms\n",
      "72:\tlearn: 0.0501944\ttotal: 1.96s\tremaining: 726ms\n",
      "73:\tlearn: 0.0496863\ttotal: 1.99s\tremaining: 699ms\n",
      "74:\tlearn: 0.0491283\ttotal: 2.02s\tremaining: 672ms\n",
      "75:\tlearn: 0.0484275\ttotal: 2.04s\tremaining: 644ms\n",
      "76:\tlearn: 0.0476085\ttotal: 2.06s\tremaining: 617ms\n",
      "77:\tlearn: 0.0472579\ttotal: 2.09s\tremaining: 588ms\n",
      "78:\tlearn: 0.0468008\ttotal: 2.11s\tremaining: 560ms\n",
      "79:\tlearn: 0.0463506\ttotal: 2.13s\tremaining: 533ms\n",
      "80:\tlearn: 0.0461345\ttotal: 2.15s\tremaining: 505ms\n",
      "81:\tlearn: 0.0454789\ttotal: 2.18s\tremaining: 478ms\n",
      "82:\tlearn: 0.0449888\ttotal: 2.2s\tremaining: 451ms\n",
      "83:\tlearn: 0.0446690\ttotal: 2.22s\tremaining: 423ms\n",
      "84:\tlearn: 0.0443304\ttotal: 2.24s\tremaining: 396ms\n",
      "85:\tlearn: 0.0441442\ttotal: 2.26s\tremaining: 368ms\n",
      "86:\tlearn: 0.0436734\ttotal: 2.29s\tremaining: 342ms\n",
      "87:\tlearn: 0.0433487\ttotal: 2.31s\tremaining: 315ms\n",
      "88:\tlearn: 0.0428227\ttotal: 2.34s\tremaining: 289ms\n",
      "89:\tlearn: 0.0425648\ttotal: 2.36s\tremaining: 262ms\n",
      "90:\tlearn: 0.0421617\ttotal: 2.38s\tremaining: 236ms\n",
      "91:\tlearn: 0.0418486\ttotal: 2.41s\tremaining: 209ms\n",
      "92:\tlearn: 0.0416878\ttotal: 2.43s\tremaining: 183ms\n",
      "93:\tlearn: 0.0412696\ttotal: 2.45s\tremaining: 156ms\n",
      "94:\tlearn: 0.0408796\ttotal: 2.47s\tremaining: 130ms\n",
      "95:\tlearn: 0.0407201\ttotal: 2.49s\tremaining: 104ms\n",
      "96:\tlearn: 0.0405865\ttotal: 2.51s\tremaining: 77.6ms\n",
      "97:\tlearn: 0.0402813\ttotal: 2.53s\tremaining: 51.7ms\n",
      "98:\tlearn: 0.0400327\ttotal: 2.55s\tremaining: 25.8ms\n",
      "99:\tlearn: 0.0397842\ttotal: 2.58s\tremaining: 0us\n",
      "0:\tlearn: 1.2814873\ttotal: 45ms\tremaining: 4.46s\n",
      "1:\tlearn: 1.0859168\ttotal: 74.8ms\tremaining: 3.67s\n",
      "2:\tlearn: 0.9409406\ttotal: 105ms\tremaining: 3.39s\n",
      "3:\tlearn: 0.8257809\ttotal: 134ms\tremaining: 3.22s\n",
      "4:\tlearn: 0.7342064\ttotal: 163ms\tremaining: 3.1s\n",
      "5:\tlearn: 0.6603161\ttotal: 195ms\tremaining: 3.05s\n",
      "6:\tlearn: 0.5961951\ttotal: 225ms\tremaining: 2.99s\n",
      "7:\tlearn: 0.5416363\ttotal: 255ms\tremaining: 2.94s\n",
      "8:\tlearn: 0.4949415\ttotal: 281ms\tremaining: 2.84s\n",
      "9:\tlearn: 0.4557863\ttotal: 306ms\tremaining: 2.75s\n",
      "10:\tlearn: 0.4185975\ttotal: 337ms\tremaining: 2.73s\n",
      "11:\tlearn: 0.3856808\ttotal: 366ms\tremaining: 2.68s\n",
      "12:\tlearn: 0.3559412\ttotal: 397ms\tremaining: 2.65s\n",
      "13:\tlearn: 0.3301298\ttotal: 420ms\tremaining: 2.58s\n",
      "14:\tlearn: 0.3058192\ttotal: 452ms\tremaining: 2.56s\n",
      "15:\tlearn: 0.2832247\ttotal: 485ms\tremaining: 2.55s\n",
      "16:\tlearn: 0.2636838\ttotal: 519ms\tremaining: 2.53s\n",
      "17:\tlearn: 0.2473297\ttotal: 547ms\tremaining: 2.49s\n",
      "18:\tlearn: 0.2320865\ttotal: 576ms\tremaining: 2.45s\n",
      "19:\tlearn: 0.2170348\ttotal: 605ms\tremaining: 2.42s\n",
      "20:\tlearn: 0.2048346\ttotal: 630ms\tremaining: 2.37s\n",
      "21:\tlearn: 0.1934058\ttotal: 658ms\tremaining: 2.33s\n",
      "22:\tlearn: 0.1833131\ttotal: 685ms\tremaining: 2.29s\n",
      "23:\tlearn: 0.1738331\ttotal: 717ms\tremaining: 2.27s\n",
      "24:\tlearn: 0.1657050\ttotal: 745ms\tremaining: 2.23s\n",
      "25:\tlearn: 0.1582447\ttotal: 773ms\tremaining: 2.2s\n",
      "26:\tlearn: 0.1517565\ttotal: 799ms\tremaining: 2.16s\n",
      "27:\tlearn: 0.1452235\ttotal: 823ms\tremaining: 2.12s\n",
      "28:\tlearn: 0.1385510\ttotal: 850ms\tremaining: 2.08s\n",
      "29:\tlearn: 0.1325743\ttotal: 875ms\tremaining: 2.04s\n",
      "30:\tlearn: 0.1274398\ttotal: 901ms\tremaining: 2s\n",
      "31:\tlearn: 0.1220018\ttotal: 929ms\tremaining: 1.97s\n",
      "32:\tlearn: 0.1181869\ttotal: 952ms\tremaining: 1.93s\n",
      "33:\tlearn: 0.1137346\ttotal: 978ms\tremaining: 1.9s\n",
      "34:\tlearn: 0.1105701\ttotal: 1s\tremaining: 1.86s\n",
      "35:\tlearn: 0.1060956\ttotal: 1.03s\tremaining: 1.83s\n",
      "36:\tlearn: 0.1031238\ttotal: 1.05s\tremaining: 1.8s\n",
      "37:\tlearn: 0.0994873\ttotal: 1.08s\tremaining: 1.76s\n",
      "38:\tlearn: 0.0955618\ttotal: 1.11s\tremaining: 1.73s\n",
      "39:\tlearn: 0.0936563\ttotal: 1.13s\tremaining: 1.69s\n",
      "40:\tlearn: 0.0909283\ttotal: 1.16s\tremaining: 1.66s\n",
      "41:\tlearn: 0.0878994\ttotal: 1.18s\tremaining: 1.63s\n",
      "42:\tlearn: 0.0854011\ttotal: 1.21s\tremaining: 1.6s\n",
      "43:\tlearn: 0.0833914\ttotal: 1.23s\tremaining: 1.57s\n",
      "44:\tlearn: 0.0810219\ttotal: 1.26s\tremaining: 1.54s\n",
      "45:\tlearn: 0.0794085\ttotal: 1.28s\tremaining: 1.5s\n",
      "46:\tlearn: 0.0781025\ttotal: 1.3s\tremaining: 1.47s\n",
      "47:\tlearn: 0.0763469\ttotal: 1.33s\tremaining: 1.44s\n",
      "48:\tlearn: 0.0744225\ttotal: 1.35s\tremaining: 1.41s\n",
      "49:\tlearn: 0.0725370\ttotal: 1.38s\tremaining: 1.38s\n",
      "50:\tlearn: 0.0711740\ttotal: 1.4s\tremaining: 1.34s\n",
      "51:\tlearn: 0.0694414\ttotal: 1.43s\tremaining: 1.32s\n",
      "52:\tlearn: 0.0682654\ttotal: 1.45s\tremaining: 1.29s\n",
      "53:\tlearn: 0.0666825\ttotal: 1.48s\tremaining: 1.26s\n",
      "54:\tlearn: 0.0659505\ttotal: 1.5s\tremaining: 1.22s\n",
      "55:\tlearn: 0.0642240\ttotal: 1.52s\tremaining: 1.2s\n",
      "56:\tlearn: 0.0628442\ttotal: 1.55s\tremaining: 1.17s\n",
      "57:\tlearn: 0.0617848\ttotal: 1.57s\tremaining: 1.14s\n",
      "58:\tlearn: 0.0609592\ttotal: 1.6s\tremaining: 1.11s\n",
      "59:\tlearn: 0.0602860\ttotal: 1.62s\tremaining: 1.08s\n",
      "60:\tlearn: 0.0591421\ttotal: 1.64s\tremaining: 1.05s\n",
      "61:\tlearn: 0.0579059\ttotal: 1.67s\tremaining: 1.02s\n",
      "62:\tlearn: 0.0567323\ttotal: 1.69s\tremaining: 995ms\n",
      "63:\tlearn: 0.0557498\ttotal: 1.72s\tremaining: 967ms\n",
      "64:\tlearn: 0.0549421\ttotal: 1.75s\tremaining: 940ms\n",
      "65:\tlearn: 0.0535980\ttotal: 1.77s\tremaining: 913ms\n",
      "66:\tlearn: 0.0530223\ttotal: 1.8s\tremaining: 887ms\n",
      "67:\tlearn: 0.0524056\ttotal: 1.82s\tremaining: 857ms\n",
      "68:\tlearn: 0.0512695\ttotal: 1.85s\tremaining: 832ms\n",
      "69:\tlearn: 0.0503820\ttotal: 1.88s\tremaining: 805ms\n",
      "70:\tlearn: 0.0499820\ttotal: 1.9s\tremaining: 776ms\n",
      "71:\tlearn: 0.0491975\ttotal: 1.93s\tremaining: 749ms\n",
      "72:\tlearn: 0.0486810\ttotal: 1.95s\tremaining: 720ms\n",
      "73:\tlearn: 0.0482335\ttotal: 1.97s\tremaining: 692ms\n",
      "74:\tlearn: 0.0474881\ttotal: 2s\tremaining: 665ms\n",
      "75:\tlearn: 0.0471092\ttotal: 2.02s\tremaining: 637ms\n",
      "76:\tlearn: 0.0468514\ttotal: 2.04s\tremaining: 608ms\n",
      "77:\tlearn: 0.0463863\ttotal: 2.06s\tremaining: 581ms\n",
      "78:\tlearn: 0.0457129\ttotal: 2.08s\tremaining: 554ms\n",
      "79:\tlearn: 0.0453896\ttotal: 2.1s\tremaining: 526ms\n",
      "80:\tlearn: 0.0451821\ttotal: 2.13s\tremaining: 498ms\n",
      "81:\tlearn: 0.0444679\ttotal: 2.15s\tremaining: 472ms\n",
      "82:\tlearn: 0.0440686\ttotal: 2.17s\tremaining: 445ms\n",
      "83:\tlearn: 0.0432873\ttotal: 2.2s\tremaining: 419ms\n",
      "84:\tlearn: 0.0430706\ttotal: 2.22s\tremaining: 392ms\n",
      "85:\tlearn: 0.0425152\ttotal: 2.25s\tremaining: 366ms\n",
      "86:\tlearn: 0.0419922\ttotal: 2.27s\tremaining: 339ms\n",
      "87:\tlearn: 0.0416166\ttotal: 2.29s\tremaining: 313ms\n",
      "88:\tlearn: 0.0414072\ttotal: 2.31s\tremaining: 286ms\n",
      "89:\tlearn: 0.0412208\ttotal: 2.33s\tremaining: 259ms\n",
      "90:\tlearn: 0.0406029\ttotal: 2.36s\tremaining: 233ms\n",
      "91:\tlearn: 0.0401395\ttotal: 2.38s\tremaining: 207ms\n",
      "92:\tlearn: 0.0399041\ttotal: 2.4s\tremaining: 181ms\n",
      "93:\tlearn: 0.0397568\ttotal: 2.43s\tremaining: 155ms\n",
      "94:\tlearn: 0.0393532\ttotal: 2.45s\tremaining: 129ms\n",
      "95:\tlearn: 0.0389765\ttotal: 2.47s\tremaining: 103ms\n",
      "96:\tlearn: 0.0385841\ttotal: 2.5s\tremaining: 77.2ms\n",
      "97:\tlearn: 0.0383227\ttotal: 2.52s\tremaining: 51.4ms\n",
      "98:\tlearn: 0.0380841\ttotal: 2.54s\tremaining: 25.6ms\n",
      "99:\tlearn: 0.0378259\ttotal: 2.57s\tremaining: 0us\n",
      "0:\tlearn: 1.2830776\ttotal: 36.1ms\tremaining: 3.57s\n",
      "1:\tlearn: 1.0829549\ttotal: 66.9ms\tremaining: 3.27s\n",
      "2:\tlearn: 0.9353468\ttotal: 95.4ms\tremaining: 3.08s\n",
      "3:\tlearn: 0.8228633\ttotal: 127ms\tremaining: 3.04s\n",
      "4:\tlearn: 0.7314785\ttotal: 157ms\tremaining: 2.97s\n",
      "5:\tlearn: 0.6570095\ttotal: 184ms\tremaining: 2.87s\n",
      "6:\tlearn: 0.5922580\ttotal: 214ms\tremaining: 2.84s\n",
      "7:\tlearn: 0.5379365\ttotal: 245ms\tremaining: 2.82s\n",
      "8:\tlearn: 0.4909762\ttotal: 275ms\tremaining: 2.78s\n",
      "9:\tlearn: 0.4523310\ttotal: 300ms\tremaining: 2.7s\n",
      "10:\tlearn: 0.4168074\ttotal: 328ms\tremaining: 2.65s\n",
      "11:\tlearn: 0.3844031\ttotal: 354ms\tremaining: 2.6s\n",
      "12:\tlearn: 0.3571935\ttotal: 381ms\tremaining: 2.55s\n",
      "13:\tlearn: 0.3314702\ttotal: 405ms\tremaining: 2.49s\n",
      "14:\tlearn: 0.3076351\ttotal: 435ms\tremaining: 2.46s\n",
      "15:\tlearn: 0.2872094\ttotal: 462ms\tremaining: 2.42s\n",
      "16:\tlearn: 0.2674436\ttotal: 493ms\tremaining: 2.41s\n",
      "17:\tlearn: 0.2494964\ttotal: 522ms\tremaining: 2.38s\n",
      "18:\tlearn: 0.2328709\ttotal: 554ms\tremaining: 2.36s\n",
      "19:\tlearn: 0.2184038\ttotal: 587ms\tremaining: 2.35s\n",
      "20:\tlearn: 0.2052962\ttotal: 615ms\tremaining: 2.31s\n",
      "21:\tlearn: 0.1937352\ttotal: 645ms\tremaining: 2.29s\n",
      "22:\tlearn: 0.1835655\ttotal: 673ms\tremaining: 2.25s\n",
      "23:\tlearn: 0.1743091\ttotal: 700ms\tremaining: 2.22s\n",
      "24:\tlearn: 0.1660714\ttotal: 726ms\tremaining: 2.18s\n",
      "25:\tlearn: 0.1577307\ttotal: 754ms\tremaining: 2.14s\n",
      "26:\tlearn: 0.1515726\ttotal: 776ms\tremaining: 2.1s\n",
      "27:\tlearn: 0.1448185\ttotal: 801ms\tremaining: 2.06s\n",
      "28:\tlearn: 0.1383117\ttotal: 830ms\tremaining: 2.03s\n",
      "29:\tlearn: 0.1326948\ttotal: 856ms\tremaining: 2s\n",
      "30:\tlearn: 0.1271133\ttotal: 885ms\tremaining: 1.97s\n",
      "31:\tlearn: 0.1218779\ttotal: 912ms\tremaining: 1.94s\n",
      "32:\tlearn: 0.1177880\ttotal: 938ms\tremaining: 1.9s\n",
      "33:\tlearn: 0.1141185\ttotal: 962ms\tremaining: 1.87s\n",
      "34:\tlearn: 0.1098135\ttotal: 989ms\tremaining: 1.83s\n",
      "35:\tlearn: 0.1061842\ttotal: 1.01s\tremaining: 1.8s\n",
      "36:\tlearn: 0.1028532\ttotal: 1.04s\tremaining: 1.77s\n",
      "37:\tlearn: 0.0986563\ttotal: 1.07s\tremaining: 1.75s\n",
      "38:\tlearn: 0.0956830\ttotal: 1.1s\tremaining: 1.72s\n",
      "39:\tlearn: 0.0927312\ttotal: 1.13s\tremaining: 1.69s\n",
      "40:\tlearn: 0.0908380\ttotal: 1.15s\tremaining: 1.66s\n",
      "41:\tlearn: 0.0883638\ttotal: 1.18s\tremaining: 1.62s\n",
      "42:\tlearn: 0.0862646\ttotal: 1.2s\tremaining: 1.59s\n",
      "43:\tlearn: 0.0838606\ttotal: 1.22s\tremaining: 1.56s\n",
      "44:\tlearn: 0.0816282\ttotal: 1.25s\tremaining: 1.53s\n",
      "45:\tlearn: 0.0799570\ttotal: 1.27s\tremaining: 1.49s\n",
      "46:\tlearn: 0.0778884\ttotal: 1.3s\tremaining: 1.46s\n",
      "47:\tlearn: 0.0758952\ttotal: 1.32s\tremaining: 1.43s\n",
      "48:\tlearn: 0.0743503\ttotal: 1.35s\tremaining: 1.4s\n",
      "49:\tlearn: 0.0722364\ttotal: 1.37s\tremaining: 1.37s\n",
      "50:\tlearn: 0.0707027\ttotal: 1.4s\tremaining: 1.34s\n",
      "51:\tlearn: 0.0691344\ttotal: 1.42s\tremaining: 1.31s\n",
      "52:\tlearn: 0.0679431\ttotal: 1.45s\tremaining: 1.28s\n",
      "53:\tlearn: 0.0670148\ttotal: 1.47s\tremaining: 1.25s\n",
      "54:\tlearn: 0.0659032\ttotal: 1.49s\tremaining: 1.22s\n",
      "55:\tlearn: 0.0644729\ttotal: 1.51s\tremaining: 1.19s\n",
      "56:\tlearn: 0.0631002\ttotal: 1.54s\tremaining: 1.16s\n",
      "57:\tlearn: 0.0619724\ttotal: 1.56s\tremaining: 1.13s\n",
      "58:\tlearn: 0.0611812\ttotal: 1.58s\tremaining: 1.1s\n",
      "59:\tlearn: 0.0601965\ttotal: 1.61s\tremaining: 1.07s\n",
      "60:\tlearn: 0.0594523\ttotal: 1.63s\tremaining: 1.04s\n",
      "61:\tlearn: 0.0585207\ttotal: 1.65s\tremaining: 1.01s\n",
      "62:\tlearn: 0.0572192\ttotal: 1.68s\tremaining: 986ms\n",
      "63:\tlearn: 0.0564851\ttotal: 1.7s\tremaining: 957ms\n",
      "64:\tlearn: 0.0558057\ttotal: 1.72s\tremaining: 928ms\n",
      "65:\tlearn: 0.0554398\ttotal: 1.74s\tremaining: 899ms\n",
      "66:\tlearn: 0.0550016\ttotal: 1.76s\tremaining: 869ms\n",
      "67:\tlearn: 0.0541620\ttotal: 1.79s\tremaining: 843ms\n",
      "68:\tlearn: 0.0535071\ttotal: 1.81s\tremaining: 815ms\n",
      "69:\tlearn: 0.0526477\ttotal: 1.84s\tremaining: 789ms\n",
      "70:\tlearn: 0.0522657\ttotal: 1.86s\tremaining: 760ms\n",
      "71:\tlearn: 0.0515884\ttotal: 1.88s\tremaining: 732ms\n",
      "72:\tlearn: 0.0508960\ttotal: 1.9s\tremaining: 704ms\n",
      "73:\tlearn: 0.0499411\ttotal: 1.93s\tremaining: 678ms\n",
      "74:\tlearn: 0.0493270\ttotal: 1.95s\tremaining: 650ms\n",
      "75:\tlearn: 0.0486457\ttotal: 1.97s\tremaining: 624ms\n",
      "76:\tlearn: 0.0483646\ttotal: 1.99s\tremaining: 596ms\n",
      "77:\tlearn: 0.0479804\ttotal: 2.02s\tremaining: 569ms\n",
      "78:\tlearn: 0.0474148\ttotal: 2.04s\tremaining: 542ms\n",
      "79:\tlearn: 0.0469324\ttotal: 2.06s\tremaining: 515ms\n",
      "80:\tlearn: 0.0468090\ttotal: 2.08s\tremaining: 488ms\n",
      "81:\tlearn: 0.0463519\ttotal: 2.1s\tremaining: 461ms\n",
      "82:\tlearn: 0.0459838\ttotal: 2.12s\tremaining: 435ms\n",
      "83:\tlearn: 0.0454873\ttotal: 2.15s\tremaining: 409ms\n",
      "84:\tlearn: 0.0447790\ttotal: 2.17s\tremaining: 384ms\n",
      "85:\tlearn: 0.0445939\ttotal: 2.19s\tremaining: 357ms\n",
      "86:\tlearn: 0.0443984\ttotal: 2.21s\tremaining: 331ms\n",
      "87:\tlearn: 0.0438835\ttotal: 2.24s\tremaining: 305ms\n",
      "88:\tlearn: 0.0435819\ttotal: 2.26s\tremaining: 280ms\n",
      "89:\tlearn: 0.0434088\ttotal: 2.28s\tremaining: 254ms\n",
      "90:\tlearn: 0.0431635\ttotal: 2.3s\tremaining: 228ms\n",
      "91:\tlearn: 0.0428744\ttotal: 2.32s\tremaining: 202ms\n",
      "92:\tlearn: 0.0424672\ttotal: 2.35s\tremaining: 177ms\n",
      "93:\tlearn: 0.0419845\ttotal: 2.37s\tremaining: 151ms\n",
      "94:\tlearn: 0.0417569\ttotal: 2.39s\tremaining: 126ms\n",
      "95:\tlearn: 0.0414068\ttotal: 2.42s\tremaining: 101ms\n",
      "96:\tlearn: 0.0410239\ttotal: 2.44s\tremaining: 75.6ms\n",
      "97:\tlearn: 0.0406319\ttotal: 2.47s\tremaining: 50.4ms\n",
      "98:\tlearn: 0.0404259\ttotal: 2.49s\tremaining: 25.2ms\n",
      "99:\tlearn: 0.0401787\ttotal: 2.51s\tremaining: 0us\n",
      "0:\tlearn: 1.2976015\ttotal: 48.4ms\tremaining: 4.79s\n",
      "1:\tlearn: 1.1022747\ttotal: 78.2ms\tremaining: 3.83s\n",
      "2:\tlearn: 0.9516341\ttotal: 109ms\tremaining: 3.51s\n",
      "3:\tlearn: 0.8338716\ttotal: 143ms\tremaining: 3.42s\n",
      "4:\tlearn: 0.7410486\ttotal: 172ms\tremaining: 3.26s\n",
      "5:\tlearn: 0.6671252\ttotal: 203ms\tremaining: 3.18s\n",
      "6:\tlearn: 0.6022883\ttotal: 233ms\tremaining: 3.09s\n",
      "7:\tlearn: 0.5469161\ttotal: 263ms\tremaining: 3.03s\n",
      "8:\tlearn: 0.4991545\ttotal: 292ms\tremaining: 2.96s\n",
      "9:\tlearn: 0.4575798\ttotal: 324ms\tremaining: 2.91s\n",
      "10:\tlearn: 0.4202057\ttotal: 354ms\tremaining: 2.86s\n",
      "11:\tlearn: 0.3868066\ttotal: 386ms\tremaining: 2.83s\n",
      "12:\tlearn: 0.3597359\ttotal: 412ms\tremaining: 2.76s\n",
      "13:\tlearn: 0.3331888\ttotal: 442ms\tremaining: 2.71s\n",
      "14:\tlearn: 0.3083132\ttotal: 475ms\tremaining: 2.69s\n",
      "15:\tlearn: 0.2880799\ttotal: 500ms\tremaining: 2.63s\n",
      "16:\tlearn: 0.2688162\ttotal: 534ms\tremaining: 2.61s\n",
      "17:\tlearn: 0.2512301\ttotal: 563ms\tremaining: 2.56s\n",
      "18:\tlearn: 0.2342615\ttotal: 592ms\tremaining: 2.52s\n",
      "19:\tlearn: 0.2198004\ttotal: 621ms\tremaining: 2.48s\n",
      "20:\tlearn: 0.2075520\ttotal: 652ms\tremaining: 2.45s\n",
      "21:\tlearn: 0.1958345\ttotal: 681ms\tremaining: 2.41s\n",
      "22:\tlearn: 0.1850173\ttotal: 711ms\tremaining: 2.38s\n",
      "23:\tlearn: 0.1755485\ttotal: 738ms\tremaining: 2.34s\n",
      "24:\tlearn: 0.1664032\ttotal: 766ms\tremaining: 2.3s\n",
      "25:\tlearn: 0.1588350\ttotal: 796ms\tremaining: 2.27s\n",
      "26:\tlearn: 0.1516355\ttotal: 821ms\tremaining: 2.22s\n",
      "27:\tlearn: 0.1453077\ttotal: 851ms\tremaining: 2.19s\n",
      "28:\tlearn: 0.1397027\ttotal: 878ms\tremaining: 2.15s\n",
      "29:\tlearn: 0.1338156\ttotal: 907ms\tremaining: 2.12s\n",
      "30:\tlearn: 0.1284053\ttotal: 933ms\tremaining: 2.08s\n",
      "31:\tlearn: 0.1227716\ttotal: 960ms\tremaining: 2.04s\n",
      "32:\tlearn: 0.1186364\ttotal: 985ms\tremaining: 2s\n",
      "33:\tlearn: 0.1140155\ttotal: 1.01s\tremaining: 1.97s\n",
      "34:\tlearn: 0.1102655\ttotal: 1.04s\tremaining: 1.94s\n",
      "35:\tlearn: 0.1064938\ttotal: 1.07s\tremaining: 1.9s\n",
      "36:\tlearn: 0.1030910\ttotal: 1.09s\tremaining: 1.86s\n",
      "37:\tlearn: 0.1005653\ttotal: 1.12s\tremaining: 1.82s\n",
      "38:\tlearn: 0.0973722\ttotal: 1.15s\tremaining: 1.79s\n",
      "39:\tlearn: 0.0947319\ttotal: 1.17s\tremaining: 1.76s\n",
      "40:\tlearn: 0.0924484\ttotal: 1.2s\tremaining: 1.72s\n",
      "41:\tlearn: 0.0900207\ttotal: 1.22s\tremaining: 1.69s\n",
      "42:\tlearn: 0.0878689\ttotal: 1.25s\tremaining: 1.65s\n",
      "43:\tlearn: 0.0861891\ttotal: 1.27s\tremaining: 1.62s\n",
      "44:\tlearn: 0.0838778\ttotal: 1.3s\tremaining: 1.59s\n",
      "45:\tlearn: 0.0813689\ttotal: 1.33s\tremaining: 1.56s\n",
      "46:\tlearn: 0.0791242\ttotal: 1.35s\tremaining: 1.53s\n",
      "47:\tlearn: 0.0770438\ttotal: 1.38s\tremaining: 1.49s\n",
      "48:\tlearn: 0.0755798\ttotal: 1.4s\tremaining: 1.46s\n",
      "49:\tlearn: 0.0738407\ttotal: 1.43s\tremaining: 1.43s\n",
      "50:\tlearn: 0.0727678\ttotal: 1.45s\tremaining: 1.39s\n",
      "51:\tlearn: 0.0706135\ttotal: 1.48s\tremaining: 1.36s\n",
      "52:\tlearn: 0.0693024\ttotal: 1.5s\tremaining: 1.33s\n",
      "53:\tlearn: 0.0683422\ttotal: 1.52s\tremaining: 1.3s\n",
      "54:\tlearn: 0.0672191\ttotal: 1.54s\tremaining: 1.26s\n",
      "55:\tlearn: 0.0653742\ttotal: 1.57s\tremaining: 1.23s\n",
      "56:\tlearn: 0.0640476\ttotal: 1.59s\tremaining: 1.2s\n",
      "57:\tlearn: 0.0625653\ttotal: 1.62s\tremaining: 1.17s\n",
      "58:\tlearn: 0.0619289\ttotal: 1.64s\tremaining: 1.14s\n",
      "59:\tlearn: 0.0608784\ttotal: 1.66s\tremaining: 1.11s\n",
      "60:\tlearn: 0.0599743\ttotal: 1.69s\tremaining: 1.08s\n",
      "61:\tlearn: 0.0587142\ttotal: 1.71s\tremaining: 1.05s\n",
      "62:\tlearn: 0.0576625\ttotal: 1.74s\tremaining: 1.02s\n",
      "63:\tlearn: 0.0567733\ttotal: 1.76s\tremaining: 992ms\n",
      "64:\tlearn: 0.0559294\ttotal: 1.79s\tremaining: 963ms\n",
      "65:\tlearn: 0.0552793\ttotal: 1.81s\tremaining: 932ms\n",
      "66:\tlearn: 0.0544216\ttotal: 1.83s\tremaining: 904ms\n",
      "67:\tlearn: 0.0540792\ttotal: 1.85s\tremaining: 873ms\n",
      "68:\tlearn: 0.0532780\ttotal: 1.88s\tremaining: 844ms\n",
      "69:\tlearn: 0.0527423\ttotal: 1.9s\tremaining: 815ms\n",
      "70:\tlearn: 0.0521370\ttotal: 1.92s\tremaining: 786ms\n",
      "71:\tlearn: 0.0514677\ttotal: 1.95s\tremaining: 758ms\n",
      "72:\tlearn: 0.0504665\ttotal: 1.97s\tremaining: 729ms\n",
      "73:\tlearn: 0.0502266\ttotal: 1.99s\tremaining: 700ms\n",
      "74:\tlearn: 0.0497221\ttotal: 2.01s\tremaining: 672ms\n",
      "75:\tlearn: 0.0490687\ttotal: 2.04s\tremaining: 644ms\n",
      "76:\tlearn: 0.0485522\ttotal: 2.06s\tremaining: 616ms\n",
      "77:\tlearn: 0.0481306\ttotal: 2.08s\tremaining: 588ms\n",
      "78:\tlearn: 0.0476122\ttotal: 2.11s\tremaining: 560ms\n",
      "79:\tlearn: 0.0473197\ttotal: 2.13s\tremaining: 533ms\n",
      "80:\tlearn: 0.0470528\ttotal: 2.15s\tremaining: 505ms\n",
      "81:\tlearn: 0.0468359\ttotal: 2.17s\tremaining: 477ms\n",
      "82:\tlearn: 0.0464564\ttotal: 2.19s\tremaining: 450ms\n",
      "83:\tlearn: 0.0460510\ttotal: 2.22s\tremaining: 423ms\n",
      "84:\tlearn: 0.0454665\ttotal: 2.24s\tremaining: 396ms\n",
      "85:\tlearn: 0.0446421\ttotal: 2.27s\tremaining: 369ms\n",
      "86:\tlearn: 0.0439286\ttotal: 2.29s\tremaining: 342ms\n",
      "87:\tlearn: 0.0436534\ttotal: 2.31s\tremaining: 316ms\n",
      "88:\tlearn: 0.0435300\ttotal: 2.33s\tremaining: 288ms\n",
      "89:\tlearn: 0.0430800\ttotal: 2.36s\tremaining: 262ms\n",
      "90:\tlearn: 0.0428758\ttotal: 2.38s\tremaining: 235ms\n",
      "91:\tlearn: 0.0423903\ttotal: 2.4s\tremaining: 209ms\n",
      "92:\tlearn: 0.0421169\ttotal: 2.43s\tremaining: 183ms\n",
      "93:\tlearn: 0.0415509\ttotal: 2.45s\tremaining: 157ms\n",
      "94:\tlearn: 0.0411831\ttotal: 2.48s\tremaining: 130ms\n",
      "95:\tlearn: 0.0408647\ttotal: 2.5s\tremaining: 104ms\n",
      "96:\tlearn: 0.0406049\ttotal: 2.52s\tremaining: 78.1ms\n",
      "97:\tlearn: 0.0403672\ttotal: 2.54s\tremaining: 51.9ms\n",
      "98:\tlearn: 0.0401560\ttotal: 2.56s\tremaining: 25.9ms\n",
      "99:\tlearn: 0.0396480\ttotal: 2.59s\tremaining: 0us\n",
      "0:\tlearn: 1.2951068\ttotal: 38.7ms\tremaining: 3.83s\n",
      "1:\tlearn: 1.0990403\ttotal: 69.8ms\tremaining: 3.42s\n",
      "2:\tlearn: 0.9517829\ttotal: 102ms\tremaining: 3.28s\n",
      "3:\tlearn: 0.8303300\ttotal: 137ms\tremaining: 3.28s\n",
      "4:\tlearn: 0.7373239\ttotal: 167ms\tremaining: 3.18s\n",
      "5:\tlearn: 0.6610209\ttotal: 197ms\tremaining: 3.09s\n",
      "6:\tlearn: 0.5946045\ttotal: 230ms\tremaining: 3.06s\n",
      "7:\tlearn: 0.5401258\ttotal: 263ms\tremaining: 3.02s\n",
      "8:\tlearn: 0.4922234\ttotal: 294ms\tremaining: 2.98s\n",
      "9:\tlearn: 0.4510443\ttotal: 326ms\tremaining: 2.93s\n",
      "10:\tlearn: 0.4143030\ttotal: 360ms\tremaining: 2.91s\n",
      "11:\tlearn: 0.3813583\ttotal: 392ms\tremaining: 2.88s\n",
      "12:\tlearn: 0.3537006\ttotal: 421ms\tremaining: 2.81s\n",
      "13:\tlearn: 0.3276761\ttotal: 454ms\tremaining: 2.79s\n",
      "14:\tlearn: 0.3022238\ttotal: 489ms\tremaining: 2.77s\n",
      "15:\tlearn: 0.2812169\ttotal: 518ms\tremaining: 2.72s\n",
      "16:\tlearn: 0.2618026\ttotal: 551ms\tremaining: 2.69s\n",
      "17:\tlearn: 0.2457942\ttotal: 580ms\tremaining: 2.64s\n",
      "18:\tlearn: 0.2301989\ttotal: 613ms\tremaining: 2.61s\n",
      "19:\tlearn: 0.2162513\ttotal: 645ms\tremaining: 2.58s\n",
      "20:\tlearn: 0.2037167\ttotal: 677ms\tremaining: 2.55s\n",
      "21:\tlearn: 0.1921814\ttotal: 708ms\tremaining: 2.51s\n",
      "22:\tlearn: 0.1804570\ttotal: 739ms\tremaining: 2.47s\n",
      "23:\tlearn: 0.1718235\ttotal: 769ms\tremaining: 2.43s\n",
      "24:\tlearn: 0.1623070\ttotal: 797ms\tremaining: 2.39s\n",
      "25:\tlearn: 0.1554218\ttotal: 822ms\tremaining: 2.34s\n",
      "26:\tlearn: 0.1491109\ttotal: 847ms\tremaining: 2.29s\n",
      "27:\tlearn: 0.1426876\ttotal: 874ms\tremaining: 2.25s\n",
      "28:\tlearn: 0.1362284\ttotal: 900ms\tremaining: 2.2s\n",
      "29:\tlearn: 0.1317139\ttotal: 927ms\tremaining: 2.16s\n",
      "30:\tlearn: 0.1262242\ttotal: 955ms\tremaining: 2.13s\n",
      "31:\tlearn: 0.1213869\ttotal: 982ms\tremaining: 2.09s\n",
      "32:\tlearn: 0.1175865\ttotal: 1.01s\tremaining: 2.05s\n",
      "33:\tlearn: 0.1135571\ttotal: 1.03s\tremaining: 2.01s\n",
      "34:\tlearn: 0.1092928\ttotal: 1.06s\tremaining: 1.98s\n",
      "35:\tlearn: 0.1058732\ttotal: 1.09s\tremaining: 1.94s\n",
      "36:\tlearn: 0.1032113\ttotal: 1.11s\tremaining: 1.9s\n",
      "37:\tlearn: 0.0997384\ttotal: 1.14s\tremaining: 1.86s\n",
      "38:\tlearn: 0.0963586\ttotal: 1.18s\tremaining: 1.84s\n",
      "39:\tlearn: 0.0938486\ttotal: 1.2s\tremaining: 1.8s\n",
      "40:\tlearn: 0.0915199\ttotal: 1.22s\tremaining: 1.76s\n",
      "41:\tlearn: 0.0890125\ttotal: 1.25s\tremaining: 1.73s\n",
      "42:\tlearn: 0.0865536\ttotal: 1.27s\tremaining: 1.69s\n",
      "43:\tlearn: 0.0847062\ttotal: 1.3s\tremaining: 1.66s\n",
      "44:\tlearn: 0.0823613\ttotal: 1.33s\tremaining: 1.62s\n",
      "45:\tlearn: 0.0809693\ttotal: 1.35s\tremaining: 1.58s\n",
      "46:\tlearn: 0.0790160\ttotal: 1.37s\tremaining: 1.55s\n",
      "47:\tlearn: 0.0773780\ttotal: 1.4s\tremaining: 1.51s\n",
      "48:\tlearn: 0.0758297\ttotal: 1.42s\tremaining: 1.48s\n",
      "49:\tlearn: 0.0738815\ttotal: 1.45s\tremaining: 1.45s\n",
      "50:\tlearn: 0.0726878\ttotal: 1.48s\tremaining: 1.42s\n",
      "51:\tlearn: 0.0713857\ttotal: 1.51s\tremaining: 1.39s\n",
      "52:\tlearn: 0.0697204\ttotal: 1.53s\tremaining: 1.36s\n",
      "53:\tlearn: 0.0686418\ttotal: 1.56s\tremaining: 1.33s\n",
      "54:\tlearn: 0.0674539\ttotal: 1.59s\tremaining: 1.3s\n",
      "55:\tlearn: 0.0659784\ttotal: 1.61s\tremaining: 1.27s\n",
      "56:\tlearn: 0.0646646\ttotal: 1.64s\tremaining: 1.24s\n",
      "57:\tlearn: 0.0636527\ttotal: 1.66s\tremaining: 1.21s\n",
      "58:\tlearn: 0.0628458\ttotal: 1.69s\tremaining: 1.17s\n",
      "59:\tlearn: 0.0619653\ttotal: 1.71s\tremaining: 1.14s\n",
      "60:\tlearn: 0.0605724\ttotal: 1.74s\tremaining: 1.11s\n",
      "61:\tlearn: 0.0599422\ttotal: 1.76s\tremaining: 1.08s\n",
      "62:\tlearn: 0.0587284\ttotal: 1.79s\tremaining: 1.05s\n",
      "63:\tlearn: 0.0582979\ttotal: 1.81s\tremaining: 1.02s\n",
      "64:\tlearn: 0.0576415\ttotal: 1.83s\tremaining: 985ms\n",
      "65:\tlearn: 0.0568491\ttotal: 1.85s\tremaining: 955ms\n",
      "66:\tlearn: 0.0557108\ttotal: 1.89s\tremaining: 928ms\n",
      "67:\tlearn: 0.0546785\ttotal: 1.91s\tremaining: 900ms\n",
      "68:\tlearn: 0.0542564\ttotal: 1.93s\tremaining: 869ms\n",
      "69:\tlearn: 0.0533070\ttotal: 1.96s\tremaining: 841ms\n",
      "70:\tlearn: 0.0525440\ttotal: 1.99s\tremaining: 811ms\n",
      "71:\tlearn: 0.0523180\ttotal: 2s\tremaining: 780ms\n",
      "72:\tlearn: 0.0515909\ttotal: 2.03s\tremaining: 751ms\n",
      "73:\tlearn: 0.0507168\ttotal: 2.06s\tremaining: 723ms\n",
      "74:\tlearn: 0.0501805\ttotal: 2.08s\tremaining: 694ms\n",
      "75:\tlearn: 0.0499135\ttotal: 2.1s\tremaining: 664ms\n",
      "76:\tlearn: 0.0495518\ttotal: 2.12s\tremaining: 634ms\n",
      "77:\tlearn: 0.0490883\ttotal: 2.15s\tremaining: 605ms\n",
      "78:\tlearn: 0.0483964\ttotal: 2.17s\tremaining: 577ms\n",
      "79:\tlearn: 0.0477145\ttotal: 2.2s\tremaining: 549ms\n",
      "80:\tlearn: 0.0471910\ttotal: 2.22s\tremaining: 521ms\n",
      "81:\tlearn: 0.0465956\ttotal: 2.25s\tremaining: 493ms\n",
      "82:\tlearn: 0.0463311\ttotal: 2.27s\tremaining: 464ms\n",
      "83:\tlearn: 0.0457294\ttotal: 2.29s\tremaining: 436ms\n",
      "84:\tlearn: 0.0455164\ttotal: 2.31s\tremaining: 408ms\n",
      "85:\tlearn: 0.0447480\ttotal: 2.34s\tremaining: 381ms\n",
      "86:\tlearn: 0.0444503\ttotal: 2.36s\tremaining: 353ms\n",
      "87:\tlearn: 0.0438271\ttotal: 2.39s\tremaining: 326ms\n",
      "88:\tlearn: 0.0436545\ttotal: 2.41s\tremaining: 298ms\n",
      "89:\tlearn: 0.0430988\ttotal: 2.43s\tremaining: 270ms\n",
      "90:\tlearn: 0.0427550\ttotal: 2.46s\tremaining: 243ms\n",
      "91:\tlearn: 0.0423284\ttotal: 2.48s\tremaining: 216ms\n",
      "92:\tlearn: 0.0419508\ttotal: 2.51s\tremaining: 189ms\n",
      "93:\tlearn: 0.0417629\ttotal: 2.53s\tremaining: 161ms\n",
      "94:\tlearn: 0.0413234\ttotal: 2.55s\tremaining: 134ms\n",
      "95:\tlearn: 0.0408667\ttotal: 2.58s\tremaining: 107ms\n",
      "96:\tlearn: 0.0406543\ttotal: 2.6s\tremaining: 80.5ms\n",
      "97:\tlearn: 0.0403741\ttotal: 2.63s\tremaining: 53.6ms\n",
      "98:\tlearn: 0.0402674\ttotal: 2.65s\tremaining: 26.7ms\n",
      "99:\tlearn: 0.0399177\ttotal: 2.67s\tremaining: 0us\n",
      "0:\tlearn: 1.2831967\ttotal: 40.3ms\tremaining: 3.99s\n",
      "1:\tlearn: 1.0827982\ttotal: 72.4ms\tremaining: 3.55s\n",
      "2:\tlearn: 0.9381621\ttotal: 108ms\tremaining: 3.5s\n",
      "3:\tlearn: 0.8216186\ttotal: 142ms\tremaining: 3.41s\n",
      "4:\tlearn: 0.7339962\ttotal: 173ms\tremaining: 3.28s\n",
      "5:\tlearn: 0.6585583\ttotal: 201ms\tremaining: 3.16s\n",
      "6:\tlearn: 0.5942182\ttotal: 233ms\tremaining: 3.1s\n",
      "7:\tlearn: 0.5404728\ttotal: 264ms\tremaining: 3.04s\n",
      "8:\tlearn: 0.4921704\ttotal: 294ms\tremaining: 2.97s\n",
      "9:\tlearn: 0.4520946\ttotal: 321ms\tremaining: 2.89s\n",
      "10:\tlearn: 0.4175358\ttotal: 348ms\tremaining: 2.82s\n",
      "11:\tlearn: 0.3846056\ttotal: 379ms\tremaining: 2.78s\n",
      "12:\tlearn: 0.3567305\ttotal: 405ms\tremaining: 2.71s\n",
      "13:\tlearn: 0.3316117\ttotal: 432ms\tremaining: 2.66s\n",
      "14:\tlearn: 0.3075384\ttotal: 462ms\tremaining: 2.62s\n",
      "15:\tlearn: 0.2865668\ttotal: 493ms\tremaining: 2.59s\n",
      "16:\tlearn: 0.2683352\ttotal: 525ms\tremaining: 2.56s\n",
      "17:\tlearn: 0.2500425\ttotal: 556ms\tremaining: 2.53s\n",
      "18:\tlearn: 0.2328843\ttotal: 586ms\tremaining: 2.5s\n",
      "19:\tlearn: 0.2185549\ttotal: 617ms\tremaining: 2.47s\n",
      "20:\tlearn: 0.2068746\ttotal: 642ms\tremaining: 2.41s\n",
      "21:\tlearn: 0.1956071\ttotal: 674ms\tremaining: 2.39s\n",
      "22:\tlearn: 0.1861447\ttotal: 701ms\tremaining: 2.35s\n",
      "23:\tlearn: 0.1759398\ttotal: 728ms\tremaining: 2.3s\n",
      "24:\tlearn: 0.1667594\ttotal: 756ms\tremaining: 2.27s\n",
      "25:\tlearn: 0.1592974\ttotal: 784ms\tremaining: 2.23s\n",
      "26:\tlearn: 0.1525784\ttotal: 809ms\tremaining: 2.19s\n",
      "27:\tlearn: 0.1463362\ttotal: 835ms\tremaining: 2.15s\n",
      "28:\tlearn: 0.1398792\ttotal: 867ms\tremaining: 2.12s\n",
      "29:\tlearn: 0.1338640\ttotal: 893ms\tremaining: 2.08s\n",
      "30:\tlearn: 0.1283818\ttotal: 923ms\tremaining: 2.05s\n",
      "31:\tlearn: 0.1238690\ttotal: 947ms\tremaining: 2.01s\n",
      "32:\tlearn: 0.1192974\ttotal: 973ms\tremaining: 1.98s\n",
      "33:\tlearn: 0.1144263\ttotal: 1s\tremaining: 1.94s\n",
      "34:\tlearn: 0.1099231\ttotal: 1.03s\tremaining: 1.91s\n",
      "35:\tlearn: 0.1061929\ttotal: 1.05s\tremaining: 1.87s\n",
      "36:\tlearn: 0.1036814\ttotal: 1.07s\tremaining: 1.83s\n",
      "37:\tlearn: 0.1004319\ttotal: 1.1s\tremaining: 1.8s\n",
      "38:\tlearn: 0.0966707\ttotal: 1.14s\tremaining: 1.77s\n",
      "39:\tlearn: 0.0933533\ttotal: 1.17s\tremaining: 1.75s\n",
      "40:\tlearn: 0.0908882\ttotal: 1.19s\tremaining: 1.72s\n",
      "41:\tlearn: 0.0888748\ttotal: 1.22s\tremaining: 1.68s\n",
      "42:\tlearn: 0.0867273\ttotal: 1.25s\tremaining: 1.65s\n",
      "43:\tlearn: 0.0845954\ttotal: 1.27s\tremaining: 1.61s\n",
      "44:\tlearn: 0.0822326\ttotal: 1.3s\tremaining: 1.58s\n",
      "45:\tlearn: 0.0804494\ttotal: 1.32s\tremaining: 1.55s\n",
      "46:\tlearn: 0.0778653\ttotal: 1.35s\tremaining: 1.52s\n",
      "47:\tlearn: 0.0763265\ttotal: 1.37s\tremaining: 1.49s\n",
      "48:\tlearn: 0.0747743\ttotal: 1.4s\tremaining: 1.46s\n",
      "49:\tlearn: 0.0731216\ttotal: 1.43s\tremaining: 1.43s\n",
      "50:\tlearn: 0.0714352\ttotal: 1.45s\tremaining: 1.39s\n",
      "51:\tlearn: 0.0700114\ttotal: 1.48s\tremaining: 1.36s\n",
      "52:\tlearn: 0.0682229\ttotal: 1.5s\tremaining: 1.33s\n",
      "53:\tlearn: 0.0671558\ttotal: 1.53s\tremaining: 1.3s\n",
      "54:\tlearn: 0.0664304\ttotal: 1.55s\tremaining: 1.27s\n",
      "55:\tlearn: 0.0648175\ttotal: 1.58s\tremaining: 1.24s\n",
      "56:\tlearn: 0.0634330\ttotal: 1.6s\tremaining: 1.21s\n",
      "57:\tlearn: 0.0621202\ttotal: 1.63s\tremaining: 1.18s\n",
      "58:\tlearn: 0.0613892\ttotal: 1.65s\tremaining: 1.15s\n",
      "59:\tlearn: 0.0605538\ttotal: 1.67s\tremaining: 1.11s\n",
      "60:\tlearn: 0.0600031\ttotal: 1.69s\tremaining: 1.08s\n",
      "61:\tlearn: 0.0590306\ttotal: 1.72s\tremaining: 1.05s\n",
      "62:\tlearn: 0.0579555\ttotal: 1.74s\tremaining: 1.02s\n",
      "63:\tlearn: 0.0573681\ttotal: 1.76s\tremaining: 991ms\n",
      "64:\tlearn: 0.0568069\ttotal: 1.78s\tremaining: 959ms\n",
      "65:\tlearn: 0.0558413\ttotal: 1.8s\tremaining: 930ms\n",
      "66:\tlearn: 0.0548027\ttotal: 1.83s\tremaining: 903ms\n",
      "67:\tlearn: 0.0543427\ttotal: 1.85s\tremaining: 872ms\n",
      "68:\tlearn: 0.0531905\ttotal: 1.88s\tremaining: 846ms\n",
      "69:\tlearn: 0.0521936\ttotal: 1.91s\tremaining: 819ms\n",
      "70:\tlearn: 0.0515294\ttotal: 1.93s\tremaining: 790ms\n",
      "71:\tlearn: 0.0507098\ttotal: 1.96s\tremaining: 763ms\n",
      "72:\tlearn: 0.0503568\ttotal: 1.99s\tremaining: 735ms\n",
      "73:\tlearn: 0.0493641\ttotal: 2.02s\tremaining: 709ms\n",
      "74:\tlearn: 0.0483707\ttotal: 2.04s\tremaining: 681ms\n",
      "75:\tlearn: 0.0477296\ttotal: 2.07s\tremaining: 654ms\n",
      "76:\tlearn: 0.0472018\ttotal: 2.09s\tremaining: 626ms\n",
      "77:\tlearn: 0.0468819\ttotal: 2.12s\tremaining: 597ms\n",
      "78:\tlearn: 0.0465212\ttotal: 2.14s\tremaining: 569ms\n",
      "79:\tlearn: 0.0463704\ttotal: 2.17s\tremaining: 541ms\n",
      "80:\tlearn: 0.0459032\ttotal: 2.19s\tremaining: 514ms\n",
      "81:\tlearn: 0.0455338\ttotal: 2.22s\tremaining: 487ms\n",
      "82:\tlearn: 0.0451662\ttotal: 2.24s\tremaining: 459ms\n",
      "83:\tlearn: 0.0447391\ttotal: 2.27s\tremaining: 432ms\n",
      "84:\tlearn: 0.0443600\ttotal: 2.29s\tremaining: 404ms\n",
      "85:\tlearn: 0.0439696\ttotal: 2.31s\tremaining: 376ms\n",
      "86:\tlearn: 0.0432251\ttotal: 2.33s\tremaining: 349ms\n",
      "87:\tlearn: 0.0429317\ttotal: 2.36s\tremaining: 322ms\n",
      "88:\tlearn: 0.0426223\ttotal: 2.39s\tremaining: 295ms\n",
      "89:\tlearn: 0.0423562\ttotal: 2.41s\tremaining: 267ms\n",
      "90:\tlearn: 0.0417793\ttotal: 2.44s\tremaining: 241ms\n",
      "91:\tlearn: 0.0412077\ttotal: 2.46s\tremaining: 214ms\n",
      "92:\tlearn: 0.0407501\ttotal: 2.48s\tremaining: 187ms\n",
      "93:\tlearn: 0.0401707\ttotal: 2.51s\tremaining: 160ms\n",
      "94:\tlearn: 0.0395676\ttotal: 2.54s\tremaining: 134ms\n",
      "95:\tlearn: 0.0391032\ttotal: 2.56s\tremaining: 107ms\n",
      "96:\tlearn: 0.0387190\ttotal: 2.58s\tremaining: 80ms\n",
      "97:\tlearn: 0.0383324\ttotal: 2.61s\tremaining: 53.2ms\n",
      "98:\tlearn: 0.0380269\ttotal: 2.63s\tremaining: 26.6ms\n",
      "99:\tlearn: 0.0378425\ttotal: 2.65s\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2       3     4\n",
      "0  23067.0     16.0    45.0    45.0   5.0\n",
      "1     21.0  15946.0     5.0     1.0   0.0\n",
      "2     50.0     21.0  4133.0     4.0   0.0\n",
      "3    100.0      2.0     7.0  1046.0   3.0\n",
      "4     20.0      0.0     0.0     5.0  14.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9921447167609301\n",
      "Precision total:  0.9124306431314798\n",
      "Recall total:  0.8475906616750987\n",
      "F1 total:  0.8721606127563117\n",
      "BACC total:  0.8475906616750987\n",
      "MCC total:  0.9866988931368766\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging Combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.2945485\ttest: 1.2947203\tbest: 1.2947203 (0)\ttotal: 46.8ms\tremaining: 4.63s\n",
      "10:\tlearn: 0.4161297\ttest: 0.4171676\tbest: 0.4171676 (10)\ttotal: 339ms\tremaining: 2.74s\n",
      "20:\tlearn: 0.2065024\ttest: 0.2078715\tbest: 0.2078715 (20)\ttotal: 629ms\tremaining: 2.37s\n",
      "30:\tlearn: 0.1272195\ttest: 0.1286882\tbest: 0.1286882 (30)\ttotal: 902ms\tremaining: 2.01s\n",
      "40:\tlearn: 0.0900034\ttest: 0.0915545\tbest: 0.0915545 (40)\ttotal: 1.16s\tremaining: 1.67s\n",
      "50:\tlearn: 0.0718656\ttest: 0.0733285\tbest: 0.0733285 (50)\ttotal: 1.4s\tremaining: 1.34s\n",
      "60:\tlearn: 0.0592838\ttest: 0.0611010\tbest: 0.0611010 (60)\ttotal: 1.63s\tremaining: 1.04s\n",
      "70:\tlearn: 0.0516517\ttest: 0.0537237\tbest: 0.0537237 (70)\ttotal: 1.86s\tremaining: 759ms\n",
      "80:\tlearn: 0.0464736\ttest: 0.0488163\tbest: 0.0488163 (80)\ttotal: 2.08s\tremaining: 489ms\n",
      "90:\tlearn: 0.0421698\ttest: 0.0445757\tbest: 0.0445757 (90)\ttotal: 2.31s\tremaining: 228ms\n",
      "99:\tlearn: 0.0394618\ttest: 0.0419010\tbest: 0.0419010 (99)\ttotal: 2.5s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.04190097644\n",
      "bestIteration = 99\n",
      "\n",
      "Epoch 1/100\n",
      "650/650 [==============================] - 3s 3ms/step - loss: 0.8983 - accuracy: 0.7196 - val_loss: 0.4049 - val_accuracy: 0.8515\n",
      "Epoch 2/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.5381 - accuracy: 0.8353 - val_loss: 0.3660 - val_accuracy: 0.8578\n",
      "Epoch 3/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4861 - accuracy: 0.8419 - val_loss: 0.3531 - val_accuracy: 0.8579\n",
      "Epoch 4/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4634 - accuracy: 0.8434 - val_loss: 0.3456 - val_accuracy: 0.8601\n",
      "Epoch 5/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4543 - accuracy: 0.8431 - val_loss: 0.3437 - val_accuracy: 0.8585\n",
      "Epoch 6/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4496 - accuracy: 0.8438 - val_loss: 0.3436 - val_accuracy: 0.8593\n",
      "Epoch 7/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4442 - accuracy: 0.8447 - val_loss: 0.3438 - val_accuracy: 0.8593\n",
      "Epoch 8/100\n",
      "650/650 [==============================] - 2s 4ms/step - loss: 0.4434 - accuracy: 0.8439 - val_loss: 0.3423 - val_accuracy: 0.8591\n",
      "Epoch 9/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4345 - accuracy: 0.8453 - val_loss: 0.3397 - val_accuracy: 0.8595\n",
      "Epoch 10/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4413 - accuracy: 0.8429 - val_loss: 0.3422 - val_accuracy: 0.8587\n",
      "Epoch 11/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4379 - accuracy: 0.8447 - val_loss: 0.3424 - val_accuracy: 0.8582\n",
      "Epoch 12/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4362 - accuracy: 0.8446 - val_loss: 0.3417 - val_accuracy: 0.8589\n",
      "Epoch 13/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4354 - accuracy: 0.8446 - val_loss: 0.3397 - val_accuracy: 0.8602\n",
      "Epoch 14/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4352 - accuracy: 0.8453 - val_loss: 0.3394 - val_accuracy: 0.8588\n",
      "Epoch 15/100\n",
      "650/650 [==============================] - 2s 4ms/step - loss: 0.4352 - accuracy: 0.8450 - val_loss: 0.3415 - val_accuracy: 0.8593\n",
      "Epoch 16/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4324 - accuracy: 0.8449 - val_loss: 0.3361 - val_accuracy: 0.8600\n",
      "Epoch 17/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4354 - accuracy: 0.8443 - val_loss: 0.3405 - val_accuracy: 0.8586\n",
      "Epoch 18/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4344 - accuracy: 0.8453 - val_loss: 0.3412 - val_accuracy: 0.8588\n",
      "Epoch 19/100\n",
      "650/650 [==============================] - 2s 4ms/step - loss: 0.4357 - accuracy: 0.8447 - val_loss: 0.3456 - val_accuracy: 0.8554\n",
      "Epoch 20/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4359 - accuracy: 0.8444 - val_loss: 0.3430 - val_accuracy: 0.8557\n",
      "Epoch 21/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4376 - accuracy: 0.8439 - val_loss: 0.3398 - val_accuracy: 0.8595\n",
      "Epoch 22/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4343 - accuracy: 0.8442 - val_loss: 0.3393 - val_accuracy: 0.8582\n",
      "Epoch 23/100\n",
      "650/650 [==============================] - 2s 3ms/step - loss: 0.4328 - accuracy: 0.8449 - val_loss: 0.3412 - val_accuracy: 0.8576\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0            0        0        0        0        0        0        0        0   \n",
      "1            1        2        1        1        1        1        1        1   \n",
      "2            1        1        1        1        1        1        1        1   \n",
      "3            0        0        0        0        0        0        0        0   \n",
      "4            1        2        1        1        1        1        1        1   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "44551        0        0        0        0        0        0        0        0   \n",
      "44552        1        2        1        1        1        1        1        1   \n",
      "44553        0        0        0        0        0        0        0        0   \n",
      "44554        0        0        0        0        0        0        0        0   \n",
      "44555        0        0        0        0        0        0        0        0   \n",
      "\n",
      "       model_8  model_9  \n",
      "0            0      0.0  \n",
      "1            1      1.0  \n",
      "2            1      1.0  \n",
      "3            0      0.0  \n",
      "4            1      1.0  \n",
      "...        ...      ...  \n",
      "44551        0      0.0  \n",
      "44552        1      1.0  \n",
      "44553        0      0.0  \n",
      "44554        0      0.0  \n",
      "44555        0      0.0  \n",
      "\n",
      "[44556 rows x 10 columns]\n",
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0            0        0        0        0        0        0        0        0   \n",
      "1            1        2        1        1        1        1        1        1   \n",
      "2            1        1        1        1        1        1        1        1   \n",
      "3            0        0        0        0        0        0        0        0   \n",
      "4            1        2        1        1        1        1        1        1   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "44551        0        0        0        0        0        0        0        0   \n",
      "44552        1        2        1        1        1        1        1        1   \n",
      "44553        0        0        0        0        0        0        0        0   \n",
      "44554        0        0        0        0        0        0        0        0   \n",
      "44555        0        0        0        0        0        0        0        0   \n",
      "\n",
      "       model_8  model_9  ensemble  \n",
      "0            0      0.0         0  \n",
      "1            1      1.0         1  \n",
      "2            1      1.0         1  \n",
      "3            0      0.0         0  \n",
      "4            1      1.0         1  \n",
      "...        ...      ...       ...  \n",
      "44551        0      0.0         0  \n",
      "44552        1      1.0         1  \n",
      "44553        0      0.0         0  \n",
      "44554        0      0.0         0  \n",
      "44555        0      0.0         0  \n",
      "\n",
      "[44556 rows x 11 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0        1       2      3     4\n",
      "0  23108.0     11.0    40.0   17.0   2.0\n",
      "1    153.0  15819.0     1.0    0.0   0.0\n",
      "2     92.0     29.0  4087.0    0.0   0.0\n",
      "3    209.0      0.0     0.0  947.0   2.0\n",
      "4     18.0      2.0     0.0    5.0  14.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9869602298231439\n",
      "Precision total:  0.9444953641595678\n",
      "Recall total:  0.8270695046180883\n",
      "F1 total:  0.868900358387607\n",
      "BACC total:  0.8270695046180883\n",
      "MCC total:  0.977923580680129\n"
     ]
    }
   ],
   "source": [
    "### Bagging with many models\n",
    "##### do bootstrapping \n",
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train.assign(label = y_train)\n",
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n",
    "\n",
    "# boot_df[5]\n",
    "\n",
    "#### 2.A base model (weak model) is created on each of these subsets.\n",
    "bag_comb_pred = []\n",
    "\n",
    "# SVM\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_00 = clf.predict(X_test)\n",
    "bag_comb_pred.append(preds_svm_00)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#ADA\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train, y_train)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_00 = ada.predict(X_test)\n",
    "bag_comb_pred.append(preds_ada_00)\n",
    "\n",
    "#Catboost\n",
    "import catboost\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_00.fit(X_train_boot, y_train_boot, eval_set=(X_test, y_test), verbose=10)\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n",
    "\n",
    "#MLP\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_00 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_00)\n",
    "\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_00 = lgbm.predict(X_test)\n",
    "    bag_comb_pred.append(preds_lgbm_00)\n",
    "#KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_00=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_00.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_00.predict(X_test)\n",
    "    bag_comb_pred.append(preds_knn)\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_00 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_00 = model_rf_00.predict(X_test)\n",
    "    bag_comb_pred.append(preds_rf_00)\n",
    "#DNN\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 5\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_00 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_00.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "# dnn_00.add(tf.keras.layers.Dense(out_layer))\n",
    "dnn_00.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_00.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "from keras.callbacks import EarlyStopping\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_00.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_00.predict(X_test)\n",
    "preds_dnn_00 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_00)\n",
    "#LogReg\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg_00 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_00.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_00.predict(X_test)\n",
    "bag_comb_pred.append(preds_logreg)\n",
    "import xgboost as xgb\n",
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 5,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_00 = xgb_00.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_00)\n",
    "### 3. The models run in parallel and are independent of each other.\n",
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)\n",
    "# Voting start\n",
    "from scipy.stats import mode\n",
    "# bag_comb_pred_df = pd.DataFrame(bag_comb_pred)\n",
    "# Extract predictions columns\n",
    "\n",
    "# predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "    # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# predictions = bag_comb_pred_df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n",
    "\n",
    "name='bag_comb'\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating new dataset for level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44556 44556\n"
     ]
    }
   ],
   "source": [
    "print(len(preds_dnn_prob), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "90361     0\n",
      "55631     1\n",
      "146669    1\n",
      "103430    0\n",
      "29036     1\n",
      "         ..\n",
      "38502     0\n",
      "81420     1\n",
      "18057     0\n",
      "80221     0\n",
      "72884     0\n",
      "Name: Label, Length: 44556, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "        index  Label\n",
      "0       90361      0\n",
      "1       55631      1\n",
      "2      146669      1\n",
      "3      103430      0\n",
      "4       29036      1\n",
      "...       ...    ...\n",
      "44551   38502      0\n",
      "44552   81420      1\n",
      "44553   18057      0\n",
      "44554   80221      0\n",
      "44555   72884      0\n",
      "\n",
      "[44556 rows x 2 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0         90361\n",
       "1         55631\n",
       "2        146669\n",
       "3        103430\n",
       "4         29036\n",
       "          ...  \n",
       "44551     38502\n",
       "44552     81420\n",
       "44553     18057\n",
       "44554     80221\n",
       "44555     72884\n",
       "Name: index, Length: 44556, dtype: int64"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_from_series = y_test.to_frame()\n",
    "y_test_reset_index = df_from_series.reset_index()\n",
    "# y_test2 = y_test.reset_index(inplace=True)\n",
    "print(y_test_reset_index)\n",
    "y_test_reset_index.pop('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_reset_index.values[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_dnn_2 = []\n",
    "preds_svm_2 = []\n",
    "preds_rf_2 = []\n",
    "preds_mlp_2 = []\n",
    "preds_ada_2 = []\n",
    "preds_knn_2 = []\n",
    "preds_lgbm_2 = []\n",
    "preds_cat_2 = []\n",
    "preds_xgb_2 = []\n",
    "\n",
    "preds_lr_2 = []\n",
    "preds_dt_2 = []\n",
    "\n",
    "for i in range(0,len(preds_dnn_prob)):  \n",
    "    # print(i)\n",
    "    # print(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dnn_2.append(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_svm_2.append(preds_svm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_rf_2.append(preds_rf_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_mlp_2.append(preds_mlp_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_ada_2.append(preds_ada_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_knn_2.append(preds_knn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lgbm_2.append(preds_lgbm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_cat_2.append(preds_cat_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_xgb_2.append(preds_xgb_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lr_2.append(preds_lr_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dt_2.append(preds_dt_prob[i][y_test_reset_index.values[i][0]])\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " [1. 1. 1. ... 1. 1. 1.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n",
      "[[0.98863387 0.97566161 1.         ... 0.99997237 1.         0.        ]\n",
      " [0.97394538 0.98897664 0.9999916  ... 0.99998922 1.         1.        ]\n",
      " [0.37007144 0.3534618  0.99862046 ... 0.92529811 1.         1.        ]\n",
      " ...\n",
      " [0.96469206 0.97566161 1.         ... 0.99636005 1.         0.        ]\n",
      " [0.98238051 0.96577066 1.         ... 0.99918175 1.         0.        ]\n",
      " [0.94032717 0.88824125 0.98846981 ... 0.98019339 1.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('------------START of STRONGER LEARNER - STACK 01 -----------------', file = f)\n",
    "\n",
    "\n",
    "# Stack the vectors horizontally to create a matrix\n",
    "column_features = ['dnn','rf','lgbm','ada','knn','mlp','svm','cat','xgb','lr','dt','label']\n",
    "training_matrix2 = np.column_stack((\n",
    "                          preds_dnn_2,\n",
    "                          preds_rf_2,\n",
    "                          preds_lgbm_2,\n",
    "                          preds_ada_2,\n",
    "                          preds_knn_2, \n",
    "                          preds_mlp_2,\n",
    "                          preds_svm_2,\n",
    "                          preds_cat_2,\n",
    "                          preds_xgb_2,\n",
    "                          preds_lr_2,\n",
    "                          preds_dt_2,\n",
    "                          y_test\n",
    "                          ))\n",
    "\n",
    "training_matrix = np.column_stack((\n",
    "                          preds_dnn,\n",
    "                          preds_rf,\n",
    "                          preds_lgbm,\n",
    "                          preds_ada,\n",
    "                          preds_knn, \n",
    "                          preds_mlp,\n",
    "                          preds_svm,\n",
    "                          preds_cat,\n",
    "                          preds_xgb,\n",
    "                          preds_lr,\n",
    "                          preds_dt,\n",
    "                        #   preds\n",
    "                          y_test\n",
    "                          ))\n",
    "# Print the resulting matrix\n",
    "print(training_matrix)\n",
    "print(training_matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_level_00_0 = pd.DataFrame(training_matrix, columns=column_features)\n",
    "df_level_00_1 = pd.DataFrame(training_matrix2, columns=column_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "if feature_selection_bit == 1:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_feature_selection.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_feature_selection.csv', index=False)\n",
    "    \n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_all_features.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_all_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.988634</td>\n",
       "      <td>0.975662</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.304414</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.872094</td>\n",
       "      <td>0.996781</td>\n",
       "      <td>0.993572</td>\n",
       "      <td>0.999972</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.973945</td>\n",
       "      <td>0.988977</td>\n",
       "      <td>0.999992</td>\n",
       "      <td>0.294535</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000205</td>\n",
       "      <td>0.997279</td>\n",
       "      <td>0.997526</td>\n",
       "      <td>0.999989</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.370071</td>\n",
       "      <td>0.353462</td>\n",
       "      <td>0.998620</td>\n",
       "      <td>0.259062</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999891</td>\n",
       "      <td>0.827397</td>\n",
       "      <td>0.908873</td>\n",
       "      <td>0.964354</td>\n",
       "      <td>0.925298</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.967854</td>\n",
       "      <td>0.965771</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.307430</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999978</td>\n",
       "      <td>0.970162</td>\n",
       "      <td>0.994960</td>\n",
       "      <td>0.992909</td>\n",
       "      <td>0.993011</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.972591</td>\n",
       "      <td>0.950943</td>\n",
       "      <td>0.999960</td>\n",
       "      <td>0.287734</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.082876</td>\n",
       "      <td>0.995128</td>\n",
       "      <td>0.996060</td>\n",
       "      <td>0.999957</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44551</th>\n",
       "      <td>0.997823</td>\n",
       "      <td>0.976221</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.316745</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.968749</td>\n",
       "      <td>0.983069</td>\n",
       "      <td>0.992998</td>\n",
       "      <td>0.999872</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>0.992368</td>\n",
       "      <td>0.946726</td>\n",
       "      <td>0.999984</td>\n",
       "      <td>0.267894</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.009048</td>\n",
       "      <td>0.994106</td>\n",
       "      <td>0.996824</td>\n",
       "      <td>0.999807</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44553</th>\n",
       "      <td>0.964692</td>\n",
       "      <td>0.975662</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.305657</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.944438</td>\n",
       "      <td>0.996601</td>\n",
       "      <td>0.992909</td>\n",
       "      <td>0.996360</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44554</th>\n",
       "      <td>0.982381</td>\n",
       "      <td>0.965771</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.307430</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.958003</td>\n",
       "      <td>0.996298</td>\n",
       "      <td>0.993572</td>\n",
       "      <td>0.999182</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44555</th>\n",
       "      <td>0.940327</td>\n",
       "      <td>0.888241</td>\n",
       "      <td>0.988470</td>\n",
       "      <td>0.232780</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999996</td>\n",
       "      <td>0.997933</td>\n",
       "      <td>0.893186</td>\n",
       "      <td>0.834807</td>\n",
       "      <td>0.980193</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44556 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
       "0      0.988634  0.975662  1.000000  0.304414  1.0  1.000000  0.872094   \n",
       "1      0.973945  0.988977  0.999992  0.294535  1.0  1.000000  0.000205   \n",
       "2      0.370071  0.353462  0.998620  0.259062  1.0  0.999891  0.827397   \n",
       "3      0.967854  0.965771  1.000000  0.307430  1.0  0.999978  0.970162   \n",
       "4      0.972591  0.950943  0.999960  0.287734  1.0  1.000000  0.082876   \n",
       "...         ...       ...       ...       ...  ...       ...       ...   \n",
       "44551  0.997823  0.976221  1.000000  0.316745  1.0  1.000000  0.968749   \n",
       "44552  0.992368  0.946726  0.999984  0.267894  1.0  1.000000  0.009048   \n",
       "44553  0.964692  0.975662  1.000000  0.305657  1.0  0.999998  0.944438   \n",
       "44554  0.982381  0.965771  1.000000  0.307430  1.0  1.000000  0.958003   \n",
       "44555  0.940327  0.888241  0.988470  0.232780  1.0  0.999996  0.997933   \n",
       "\n",
       "            cat       xgb        lr   dt  label  \n",
       "0      0.996781  0.993572  0.999972  1.0    0.0  \n",
       "1      0.997279  0.997526  0.999989  1.0    1.0  \n",
       "2      0.908873  0.964354  0.925298  1.0    1.0  \n",
       "3      0.994960  0.992909  0.993011  1.0    0.0  \n",
       "4      0.995128  0.996060  0.999957  1.0    1.0  \n",
       "...         ...       ...       ...  ...    ...  \n",
       "44551  0.983069  0.992998  0.999872  1.0    0.0  \n",
       "44552  0.994106  0.996824  0.999807  1.0    1.0  \n",
       "44553  0.996601  0.992909  0.996360  1.0    0.0  \n",
       "44554  0.996298  0.993572  0.999182  1.0    0.0  \n",
       "44555  0.893186  0.834807  0.980193  1.0    0.0  \n",
       "\n",
       "[44556 rows x 12 columns]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44551</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44552</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44553</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44554</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44555</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44556 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dnn   rf  lgbm  ada  knn  mlp  svm  cat  xgb   lr   dt  label\n",
       "0      0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "1      1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "2      1.0  1.0   1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "3      0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "4      1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "...    ...  ...   ...  ...  ...  ...  ...  ...  ...  ...  ...    ...\n",
       "44551  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "44552  1.0  1.0   1.0  2.0  1.0  1.0  1.0  1.0  1.0  1.0  1.0    1.0\n",
       "44553  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "44554  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "44555  0.0  0.0   0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "\n",
       "[44556 rows x 12 columns]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_01 = df_level_01.pop('label')\n",
    "# X_01 = df_level_01\n",
    "# df_level_01 = df_level_01.assign(label = y_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_level_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split = 0.7\n",
    "\n",
    "# X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tabulate import tabulate\n",
    "\n",
    "# # Assuming data is a 110x4 list, where each row is a sublist\n",
    "# # data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "# data = [[\"\" for _ in range(3)] for _ in range(12)]\n",
    "\n",
    "# # Manually insert data at specific row and column\n",
    "# # data[0][0] = \"ADA\"\n",
    "# # data[1][0] = \"DNN\"\n",
    "# # data[2][0] = \"SVM\"\n",
    "# # data[3][0] = \"ADA\"\n",
    "# # data[4][0] = \"DNN\"\n",
    "# # data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'VOTING'\n",
    "#                 ]\n",
    "# level_00_f1 = [ada_f1_00,\n",
    "#                 svm_f1_00,\n",
    "#                 dnn_f1_00,\n",
    "#                 mlp_f1_00,\n",
    "#                 knn_f1_00,\n",
    "#                 cat_f1_00,\n",
    "#                 xgb_f1_00,\n",
    "#                 lgbm_f1_00,\n",
    "#                 rf_f1_00,\n",
    "#                 lr_f1_00,\n",
    "#                 voting_f1_00]  \n",
    "\n",
    "                 \n",
    "\n",
    "# for i in range(0,len(names_models)):\n",
    "#     data[i][0] =  names_models[i]\n",
    "#     data[i][1] = level_00_f1[i]\n",
    "#     data[i][2] = level_01_f1[i]\n",
    "\n",
    "\n",
    " \n",
    "# # data[0][1] = ada_acc_00\n",
    "# # data\n",
    "\n",
    "# # Define column headers\n",
    "# headers = [\"F1\", \"Level 00\", \"Level 01\"]\n",
    "\n",
    "# # Print the table\n",
    "# table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "# print(table)\n",
    "# with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr_acc_00 = 0 \n",
    "# voting_acc_00 = 0\n",
    "\n",
    "# lr_pre_00 = 0 \n",
    "# voting_pre_00 = 0\n",
    "\n",
    "# lr_rec_00 = 0 \n",
    "# voting_rec_00 = 0\n",
    "\n",
    "# lr_f1_00 = 0 \n",
    "# voting_f1_00 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Models   | ACC-00             | PRE-00             | REC-00             | F1-00              |\n",
      "+==========+====================+====================+====================+====================+\n",
      "| ADA      | 0.5969566388365203 | 0.6544687165542271 | 0.5843270988523745 | 0.519034372822556  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| SVM      | 0.9696786066971901 | 0.8679619811418128 | 0.7805208139961298 | 0.8127172773370175 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| DNN      | 0.85912110602388   | 0.3412053777599212 | 0.3904672187210905 | 0.3640253879559558 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| MLP      | 0.9916060687673939 | 0.9508824653701113 | 0.8698256243197143 | 0.9037176505944327 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| KNN      | 0.9906185474459107 | 0.9565767483082197 | 0.9074171479628224 | 0.9293080592110776 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| CAT      | 0.9916285124337911 | 0.9227230583537622 | 0.8569640421623295 | 0.882375896448177  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| XGB      | 0.9929526887512344 | 0.9276999429780629 | 0.8907415640865624 | 0.9064845653101837 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| LGBM     | 0.9832794685339797 | 0.7937735171740729 | 0.8825024574163578 | 0.8129592337272072 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| RF       | 0.942162671694048  | 0.7711367890423176 | 0.5866563883432252 | 0.6215575130637341 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| LR       | 0.9770625729419158 | 0.9131874423401942 | 0.8355181237257385 | 0.8637408570604199 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| DT       | 0.9949726187269953 | 0.9006103654491    | 0.9233128079581864 | 0.9233128079581864 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_svm  | 0.9677933387198133 | 0.8753526932397333 | 0.7541964704931043 | 0.7975247689707171 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_knn  | 0.9907532094442948 | 0.9506883626415827 | 0.907749702830241  | 0.9271821363087025 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_DT   | 0.9960274710476703 | 0.951536765476604  | 0.923786980516387  | 0.9368857928559633 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_LR   | 0.9781623125953856 | 0.9056559211106784 | 0.8452455632984475 | 0.8697023580089054 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_mlp  | 0.9942768650686776 | 0.9474589833614357 | 0.8998439488245275 | 0.9190841504980451 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_rf   | 0.9483795672861118 | 0.578316428205879  | 0.5748585920561518 | 0.5759563112544444 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_ada  | 0.7863138522309004 | 0.7190130421953926 | 0.616552930067735  | 0.6455379818656247 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_lgbm | 0.9957357033845049 | 0.9323939946356916 | 0.8682894938848295 | 0.8929042317678464 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_cat  | 0.9921447167609301 | 0.9124306431314798 | 0.8475906616750987 | 0.8721606127563117 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_comb | 0.9869602298231439 | 0.9444953641595678 | 0.8270695046180883 | 0.868900358387607  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "data = [[\"\" for _ in range(5)] for _ in range(24)]\n",
    "\n",
    "# Manually insert data at specific row and column\n",
    "# data[0][0] = \"ADA\"\n",
    "# data[1][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "# data[3][0] = \"ADA\"\n",
    "# data[4][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'VOTING',\n",
    "#                 '   '\n",
    "#                 ]\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'DT',\n",
    "#                 # 'VOTING',\n",
    "#                 'Bag_svm',\n",
    "#                 'Bag_knn',\n",
    "#                 'Bag_DT',\n",
    "#                 'Bag_LR',\n",
    "#                 'Bag_mlp',\n",
    "#                 # 'avg',\n",
    "#                 # 'weighed_avg'\n",
    "#                 ]\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'DT',\n",
    "                # 'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "\n",
    "                # 'avg',\n",
    "                # 'weighed_avg'\n",
    "                ]\n",
    "\n",
    "\n",
    "level_00_acc = [ada_acc_00,\n",
    "                svm_acc_00,\n",
    "                dnn_acc_00,\n",
    "                mlp_acc_00,\n",
    "                knn_acc_00,\n",
    "                cat_acc_00,\n",
    "                xgb_acc_00,\n",
    "                lgbm_acc_00,\n",
    "                rf_acc_00,\n",
    "                lr_acc_00,\n",
    "                dt_acc_00,\n",
    "                # voting_acc_00,\n",
    "                bag_svm_acc_00,\n",
    "                bag_knn_acc_00,\n",
    "                bag_dt_acc_00,\n",
    "                bag_lr_acc_00,\n",
    "                bag_mlp_acc_00,\n",
    "               \n",
    "                bag_rf_acc_00,\n",
    "                bag_ada_acc_00,\n",
    "                bag_lgbm_acc_00,\n",
    "\n",
    "                bag_cat_acc_00,\n",
    "                bag_comb_acc_00,\n",
    "               \n",
    "               \n",
    "                \n",
    "                # avg_acc_00,\n",
    "                # weighed_avg_acc_00\n",
    "                ]  \n",
    "\n",
    "                # ]  \n",
    "\n",
    "level_00_pre = [ada_pre_00,\n",
    "                svm_pre_00,\n",
    "                dnn_pre_00,\n",
    "                mlp_pre_00,\n",
    "                knn_pre_00,\n",
    "                cat_pre_00,\n",
    "                xgb_pre_00,\n",
    "                lgbm_pre_00,\n",
    "                rf_pre_00,\n",
    "                lr_pre_00,\n",
    "                dt_pre_00,\n",
    "                # voting_pre_00,\n",
    "                bag_svm_pre_00,\n",
    "                bag_knn_pre_00,\n",
    "                bag_dt_pre_00,\n",
    "                bag_lr_pre_00,\n",
    "                bag_mlp_pre_00,\n",
    "\n",
    "                bag_rf_pre_00,\n",
    "                bag_ada_pre_00,\n",
    "                bag_lgbm_pre_00,\n",
    "\n",
    "                bag_cat_pre_00,\n",
    "                bag_comb_pre_00,\n",
    "               \n",
    "                # avg_pre_00,\n",
    "                # weighed_avg_pre_00\n",
    "                ]  \n",
    "\n",
    "level_00_rec = [ada_rec_00,\n",
    "                svm_rec_00,\n",
    "                dnn_rec_00,\n",
    "                mlp_rec_00,\n",
    "                knn_rec_00,\n",
    "                cat_rec_00,\n",
    "                xgb_rec_00,\n",
    "                lgbm_rec_00,\n",
    "                rf_rec_00,\n",
    "                lr_rec_00,\n",
    "                dt_rec_00,\n",
    "                # voting_rec_00,\n",
    "                bag_svm_rec_00,\n",
    "                bag_knn_rec_00,\n",
    "                bag_dt_rec_00,\n",
    "                bag_lr_rec_00,\n",
    "                bag_mlp_rec_00,\n",
    "\n",
    "                bag_rf_rec_00,\n",
    "                bag_ada_rec_00,\n",
    "                bag_lgbm_rec_00,\n",
    "\n",
    "                bag_cat_rec_00,\n",
    "                bag_comb_rec_00,\n",
    "               \n",
    "                # avg_rec_00,\n",
    "                # weighed_avg_rec_00\n",
    "                ]  \n",
    "\n",
    "level_00_f1 = [ada_f1_00,\n",
    "                svm_f1_00,\n",
    "                dnn_f1_00,\n",
    "                mlp_f1_00,\n",
    "                knn_f1_00,\n",
    "                cat_f1_00,\n",
    "                xgb_f1_00,\n",
    "                lgbm_f1_00,\n",
    "                rf_f1_00,\n",
    "                lr_f1_00,\n",
    "                dt_rec_00,\n",
    "                # voting_f1_00,\n",
    "                bag_svm_f1_00,\n",
    "                bag_knn_f1_00,\n",
    "                bag_dt_f1_00,\n",
    "                bag_lr_f1_00,\n",
    "                bag_mlp_f1_00,\n",
    "\n",
    "                bag_rf_f1_00,\n",
    "                bag_ada_f1_00,\n",
    "                bag_lgbm_f1_00,\n",
    "\n",
    "                bag_cat_f1_00,\n",
    "                bag_comb_f1_00,\n",
    "               \n",
    "                # avg_f1_00,\n",
    "                # weighed_avg_f1_00\n",
    "                ]                   \n",
    "\n",
    "for i in range(0,len(names_models)):\n",
    "    data[i][0] =  names_models[i]\n",
    "\n",
    "    data[i][1] = level_00_acc[i]\n",
    "    # data[i][2] = level_01_acc[i]\n",
    "\n",
    "    data[i][2] = level_00_pre[i] \n",
    "    # data[i][4] = level_01_pre[i]\n",
    "\n",
    "    data[i][3] = level_00_rec[i] \n",
    "    # data[i][6] = level_01_rec[i]\n",
    "\n",
    "    data[i][4] = level_00_f1[i]\n",
    "    # data[i][8] = level_01_f1[i]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "# data[0][1] = ada_acc_00\n",
    "# data\n",
    "\n",
    "# Define column headers\n",
    "# headers = [\"Models\", \"ACC-00\", \" ACC-01\",\"PRE-00\", \" PRE-01\",\"REC-00\", \" REC-01\",\"F1-00\", \" F1-01\",]\n",
    "headers = [\"Models\", \"ACC-00\",\"PRE-00\",\"REC-00\",\"F1-00\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Models   | ACC-00             | PRE-00             | REC-00             | F1-00              |\n",
      "+==========+====================+====================+====================+====================+\n",
      "| Bag_DT   | 0.9960274710476703 | 0.951536765476604  | 0.923786980516387  | 0.9368857928559633 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| KNN      | 0.9906185474459107 | 0.9565767483082197 | 0.9074171479628224 | 0.9293080592110776 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_knn  | 0.9907532094442948 | 0.9506883626415827 | 0.907749702830241  | 0.9271821363087025 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| DT       | 0.9949726187269953 | 0.9006103654491    | 0.9233128079581864 | 0.9233128079581864 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_mlp  | 0.9942768650686776 | 0.9474589833614357 | 0.8998439488245275 | 0.9190841504980451 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| XGB      | 0.9929526887512344 | 0.9276999429780629 | 0.8907415640865624 | 0.9064845653101837 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| MLP      | 0.9916060687673939 | 0.9508824653701113 | 0.8698256243197143 | 0.9037176505944327 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_lgbm | 0.9957357033845049 | 0.9323939946356916 | 0.8682894938848295 | 0.8929042317678464 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| CAT      | 0.9916285124337911 | 0.9227230583537622 | 0.8569640421623295 | 0.882375896448177  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_cat  | 0.9921447167609301 | 0.9124306431314798 | 0.8475906616750987 | 0.8721606127563117 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_LR   | 0.9781623125953856 | 0.9056559211106784 | 0.8452455632984475 | 0.8697023580089054 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_comb | 0.9869602298231439 | 0.9444953641595678 | 0.8270695046180883 | 0.868900358387607  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| LR       | 0.9770625729419158 | 0.9131874423401942 | 0.8355181237257385 | 0.8637408570604199 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| LGBM     | 0.9832794685339797 | 0.7937735171740729 | 0.8825024574163578 | 0.8129592337272072 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| SVM      | 0.9696786066971901 | 0.8679619811418128 | 0.7805208139961298 | 0.8127172773370175 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_svm  | 0.9677933387198133 | 0.8753526932397333 | 0.7541964704931043 | 0.7975247689707171 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_ada  | 0.7863138522309004 | 0.7190130421953926 | 0.616552930067735  | 0.6455379818656247 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| RF       | 0.942162671694048  | 0.7711367890423176 | 0.5866563883432252 | 0.6215575130637341 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| Bag_rf   | 0.9483795672861118 | 0.578316428205879  | 0.5748585920561518 | 0.5759563112544444 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| ADA      | 0.5969566388365203 | 0.6544687165542271 | 0.5843270988523745 | 0.519034372822556  |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "| DNN      | 0.85912110602388   | 0.3412053777599212 | 0.3904672187210905 | 0.3640253879559558 |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n",
      "|          |                    |                    |                    |                    |\n",
      "+----------+--------------------+--------------------+--------------------+--------------------+\n"
     ]
    }
   ],
   "source": [
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_00_acc, level_00_pre, level_00_rec, level_00_f1))\n",
    "\n",
    "# Sort by F1-00 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_00_acc, sorted_level_00_pre, sorted_level_00_rec, sorted_level_00_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_00_acc[i]\n",
    "    data[i][2] = sorted_level_00_pre[i] \n",
    "    data[i][3] = sorted_level_00_rec[i] \n",
    "    data[i][4] = sorted_level_00_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-00\", \"PRE-00\", \"REC-00\", \"F1-00\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table - LEVEL 00', file = f)\n",
    "\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------------+\n",
      "| Models   |   time-00(sec) |\n",
      "+==========+================+\n",
      "| RF       |       0.493166 |\n",
      "+----------+----------------+\n",
      "| DT       |       1.48455  |\n",
      "+----------+----------------+\n",
      "| LGBM     |       2.23693  |\n",
      "+----------+----------------+\n",
      "| CAT      |       2.94721  |\n",
      "+----------+----------------+\n",
      "| LR       |       4.94028  |\n",
      "+----------+----------------+\n",
      "| Bag_rf   |       5.74825  |\n",
      "+----------+----------------+\n",
      "| ADA      |       8.40896  |\n",
      "+----------+----------------+\n",
      "| XGB      |       9.37023  |\n",
      "+----------+----------------+\n",
      "| Bag_DT   |      10.8922   |\n",
      "+----------+----------------+\n",
      "| Bag_lgbm |      21.1009   |\n",
      "+----------+----------------+\n",
      "| Bag_cat  |      28.3599   |\n",
      "+----------+----------------+\n",
      "| SVM      |      34.5938   |\n",
      "+----------+----------------+\n",
      "| Bag_LR   |      51.6708   |\n",
      "+----------+----------------+\n",
      "| Bag_ada  |      64.8717   |\n",
      "+----------+----------------+\n",
      "| DNN      |      67.2328   |\n",
      "+----------+----------------+\n",
      "| Bag_svm  |      79.7104   |\n",
      "+----------+----------------+\n",
      "| MLP      |     119.158    |\n",
      "+----------+----------------+\n",
      "| Bag_comb |     563.579    |\n",
      "+----------+----------------+\n",
      "| KNN      |     758.682    |\n",
      "+----------+----------------+\n",
      "| Bag_mlp  |     921.355    |\n",
      "+----------+----------------+\n",
      "| Bag_knn  |    4046.57     |\n",
      "+----------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "from tabulate import tabulate\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'DT',\n",
    "                # 'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                # 'avg',\n",
    "                # 'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_00_time = [\n",
    "                ada_time_00,\n",
    "                svm_time_00,\n",
    "                dnn_time_00,\n",
    "                mlp_time_00,\n",
    "                knn_time_00,\n",
    "                cat_time_00,\n",
    "                xgb_time_00,\n",
    "                lgbm_time_00,\n",
    "                rf_time_00,\n",
    "                lr_time_00,\n",
    "                dt_time_00,\n",
    "                # voting_time_00,\n",
    "                bag_svm_time_00,\n",
    "                bag_knn_time_00,\n",
    "                bag_dt_time_00,\n",
    "                bag_lr_time_00,\n",
    "                bag_mlp_time_00,\n",
    "\n",
    "                bag_rf_time_00,\n",
    "                bag_ada_time_00,\n",
    "                bag_lgbm_time_00,\n",
    "                # bag_xgb_time_00,\n",
    "                bag_cat_time_00,\n",
    "                bag_comb_time_00,\n",
    "\n",
    "                # avg_time_00,\n",
    "                # weighed_avg_time_00\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_00_time))\n",
    "\n",
    "# Sort by F1-00 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_00_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_00_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-00(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
